Move from gitlab to github

source/_static/theme_override.css

@@ -0,0 +1,13 @@
+/* override table width restrictions */
+@media screen and (min-width: 767px) {
+
+   .wy-table-responsive table td {
+      /* !important prevents the common CSS stylesheets from overriding
+         this as on RTD they are loaded after this stylesheet */
+      white-space: normal !important;
+   }
+
+   .wy-table-responsive {
+      overflow: visible !important;
+   }
+}

source/_templates/breadcrumbs.html

@@ -0,0 +1,83 @@
+{# Support for Sphinx 1.3+ page_source_suffix, but don't break old builds. #}
+
+{% if page_source_suffix %}
+{% set suffix = page_source_suffix %}
+{% else %}
+{% set suffix = source_suffix %}
+{% endif %}
+
+{% if meta is defined and meta is not none %}
+{% set check_meta = True %}
+{% else %}
+{% set check_meta = False %}
+{% endif %}
+
+{% if check_meta and 'github_url' in meta %}
+{% set display_github = True %}
+{% endif %}
+
+{% if check_meta and 'bitbucket_url' in meta %}
+{% set display_bitbucket = True %}
+{% endif %}
+
+{% if check_meta and 'gitlab_url' in meta %}
+{% set display_gitlab = True %}
+{% endif %}
+
+<div role="navigation" aria-label="breadcrumbs navigation">
+
+  <ul class="wy-breadcrumbs">
+    {% block breadcrumbs %}
+      <li><a href="{{ pathto(master_doc) }}">{{ _('CKANEXT-DCATDONL') }}</a> &raquo;</li>
+        {% for doc in parents %}
+          <li><a href="{{ doc.link|e }}">{{ doc.title }}</a> &raquo;</li>
+        {% endfor %}
+      <li>{{ title }}</li>
+    {% endblock %}
+    {% block breadcrumbs_aside %}
+      <li class="wy-breadcrumbs-aside">
+          <a href="https://readthedocs.org/projects/ckanext-dcatdonl/downloads/pdf/latest/" class="fa fa-download"> Download as PDF</a>
+        {% if hasdoc(pagename) %}
+            {% if display_github %}
+            {% if check_meta and 'github_url' in meta %}
+              <!-- User defined GitHub URL -->
+              <a href="{{ meta['github_url'] }}" class="fa fa-github"> {{ _('Edit on GitHub') }}</a>
+            {% else %}
+              <a href="https://{{ github_host|default("github.com") }}/{{ github_user }}/{{ github_repo }}/{{ theme_vcs_pageview_mode|default("blob") }}/{{ github_version }}{{ conf_py_path }}{{ pagename }}{{ suffix }}" class="fa fa-github"> {{ _('Edit on GitHub') }}</a>
+            {% endif %}
+          {% elif display_bitbucket %}
+            {% if check_meta and 'bitbucket_url' in meta %}
+              <!-- User defined Bitbucket URL -->
+              <a href="{{ meta['bitbucket_url'] }}" class="fa fa-bitbucket"> {{ _('Edit on Bitbucket') }}</a>
+            {% else %}
+              <a href="https://bitbucket.org/{{ bitbucket_user }}/{{ bitbucket_repo }}/src/{{ bitbucket_version}}{{ conf_py_path }}{{ pagename }}{{ suffix }}?mode={{ theme_vcs_pageview_mode|default("view") }}" class="fa fa-bitbucket"> {{ _('Edit on Bitbucket') }}</a>
+            {% endif %}
+          {% elif display_gitlab %}
+            {% if check_meta and 'gitlab_url' in meta %}
+              <!-- User defined GitLab URL -->
+              <a href="{{ meta['gitlab_url'] }}" class="fa fa-gitlab"> {{ _('Edit on GitLab') }}</a>
+            {% else %}
+              <a href="https://{{ gitlab_host|default("gitlab.com") }}/{{ gitlab_user }}/{{ gitlab_repo }}/{{ theme_vcs_pageview_mode|default("blob") }}/{{ gitlab_version }}{{ conf_py_path }}{{ pagename }}{{ suffix }}" class="fa fa-gitlab"> {{ _('Edit on GitLab') }}</a>
+            {% endif %}
+          {% elif show_source and source_url_prefix %}
+            <a href="{{ source_url_prefix }}{{ pagename }}{{ suffix }}">{{ _('View page source') }}</a>
+          {% elif show_source and has_source and sourcename %}
+            <a href="{{ pathto('_sources/' + sourcename, true)|e }}" rel="nofollow"> {{ _('View page source') }}</a>
+          {% endif %}
+        {% endif %}
+      </li>
+    {% endblock %}
+  </ul>
+
+  {% if (theme_prev_next_buttons_location == 'top' or theme_prev_next_buttons_location == 'both') and (next or prev) %}
+  <div class="rst-breadcrumbs-buttons" role="navigation" aria-label="breadcrumb navigation">
+      {% if next %}
+        <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}" accesskey="n">Next <span class="fa fa-arrow-circle-right"></span></a>
+      {% endif %}
+      {% if prev %}
+        <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}" accesskey="p"><span class="fa fa-arrow-circle-left"></span> Previous</a>
+      {% endif %}
+  </div>
+  {% endif %}
+  <hr/>
+</div>

source/_templates/footer.html

@@ -0,0 +1,33 @@
+<footer>
+  {% if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %}
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      {% if next %}
+        <a href="{{ next.link|e }}" class="btn btn-neutral float-right" title="{{ next.title|striptags|e }}" accesskey="n" rel="next">{{ _('Next') }} <span class="fa fa-arrow-circle-right"></span></a>
+      {% endif %}
+      {% if prev %}
+        <a href="{{ prev.link|e }}" class="btn btn-neutral" title="{{ prev.title|striptags|e }}" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> {{ _('Previous') }}</a>
+      {% endif %}
+    </div>
+  {% endif %}
+
+  <hr/>
+
+  <div role="contentinfo">
+    <p>
+    {%- if show_copyright %}
+      {%- if hasdoc('copyright') %}
+        {% trans path=pathto('copyright'), copyright=copyright|e %}&copy; <a href="{{ path }}">Copyright</a> {{ copyright }}.{% endtrans %}
+      {%- else %}
+        {% trans copyright=copyright|e %}&copy; Copyright {{ copyright }}.{% endtrans %}
+      {%- endif %}
+    {%- endif %}
+    </p>
+  </div>
+
+  {%- if show_sphinx %}
+  {% trans %}Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>{% endtrans %}.
+  {%- endif %}
+
+  {%- block extrafooter %} {% endblock %}
+
+</footer>

source/announcements.rst

@@ -0,0 +1,10 @@
+Announcements
+============================================
+
+Below you'll find important announcements regarding this CKAN extension.
+
+Scheduled update of public test application on https://dcat-ap-donl.nl
+----------------------------------------------------------------------
+
+On :code:`05/11/2018` the dcat-ap-donl.nl test environment will be unavailable while we roll out the latest changes of
+the CKAN extension.

source/changelog-20181029.rst

@@ -0,0 +1,39 @@
+Changes applied on 29/10/2018
+============================================
+
+A new version has been released, the changes are list below!
+
+- updated installation instructions.
+    - Now includes the installation of additional requirements
+    - Added instructions for targeting specific Apache Solr versions
+- Introduced instructions to setup Apache Solr in various versions
+    - Currently contains support for Apache Solr 5.4 and 7.4
+- Introduced several backwards compatible fixes for CKAN versions below 2.6
+- Included Solr optimizations, searches against Solr now include several facets by default, namely
+    - facet_referentie_data
+    - facet_access_rights
+    - facet_publisher
+    - facet_authority
+    - facet_high_value
+    - facet_basis_register
+    - facet_dataset_status
+    - facet_metadata_language
+    - facet_frequency
+    - facet_license_id
+    - facet_source_catalog
+    - facet_theme
+- Changes to the schemas:
+    - To declare a license for a package and/or resource you must now provide it in the :code:`license_id` key rather than the `license` key.
+    - The fields :code:`highvalue`, :code:`basisregister` and :code:`referentiedata` are no longer considered data.overheid system properties and are now part of the base DCAT-AP-DONL scheme
+    - The field :code:`highvalue` has been renamed to :code:`high_value`
+    - The field :code:`referentiedata` has been renamed to :code:`referentie_data`
+    - The field :code:`basisregister` has been renamed to :code:`basis_register`
+    - The field :code:`dataset_status` will now default to the URI for :code:`beschikbaar`
+    - The field :code:`high_value` will now default to :code:`false`
+    - The field :code:`referentie_data` will now default to :code:`false`
+    - The field :code:`basis_register` will now default to :code:`false`
+    - The list validation is now less strict, when a single value is provided it will silently convert this to a list of size 1 rather than returning a validation error message
+- Updated the Usage chapter to incorporate the changes to the schemas
+- The documented error messages have been updated
+- Updated the logging format of the :code:`controlled_vocabulary_updater.py`
+- Small fixes to various chapters of this documentation containing inaccuracies

source/changelog-20190308.rst

@@ -0,0 +1,6 @@
+Changes applied on 08/03/2019
+============================================
+
+- Introduced a new property to CKAN :code:`Dataset` schema: :code:`national_coverage`. This property is an optional boolean. When this property is not present in a dataset it is considered 'false'.
+- Introduced :code:`national_coverage` to Solr schema.
+- Introduced facet field :code:`facet_national_coverage` to Solr schema. 

source/changelog.rst

@@ -0,0 +1,11 @@
+Changelog
+============================================
+
+Contains the functional changelog of this CKAN extension.
+
+.. toctree::
+   :maxdepth: 3
+   :caption: Changes
+
+   changelog-20190308
+   changelog-20181029

source/conf.py

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+project = u'ckanext-dcatdonl'
+copyright = u'2018, Kennis- en Exploitatiecentrum Officiële Overheidspublicaties'
+author = u'Willem ter Berg, Textinfo B.V.'
+version = u''
+release = u''
+extensions = []
+templates_path = ['_templates']
+source_suffix = '.rst'
+master_doc = 'index'
+language = None
+exclude_patterns = []
+pygments_style = 'sphinx'
+html_theme = 'sphinx_rtd_theme'
+html_theme_options = {
+    'navigation_depth': 4,
+    'display_version': False,
+    'collapse_navigation': False
+}
+html_show_sourcelink = False
+html_context = {
+    'commit': False
+}
+html_static_path = ['_static']
+htmlhelp_basename = 'ckanext-dcatdonldoc'
+latex_elements = {
+
+}
+latex_documents = [
+    (master_doc, 'ckanext-dcatdonl.tex', u'CKANEXT-DCATDONL Documentation', u'Kennis- en Exploitatiecentrum Officiële Overheidspublicaties', 'manual'),
+]
+latex_show_urls = 'footnote'
+man_pages = [
+    (master_doc, 'ckanext-dcatdonl', u'CKANEXT-DCATDONL Documentation', [author], 1)
+]
+texinfo_documents = [
+    (master_doc, 'ckanext-dcatdonl', u'CKANEXT-DCATDONL Documentation', author, 'ckanext-dcatdonl', 'One line description of project.', 'Miscellaneous'),
+]
+
+def setup(app):
+    app.add_stylesheet('theme_override.css')

source/index.rst

@@ -0,0 +1,15 @@
+CKANEXT-DCATDONL
+============================================
+
+The CKAN extension that implements the DCAT-AP-DONL metadata standard into CKAN.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Table of Contents
+
+   summary
+   announcements
+   changelog
+   installation
+   usage
+   schema

source/installation-backgroundprocess.rst

@@ -0,0 +1,23 @@
+Setting up the background process
+===================================================================================================
+
+In order for the ckanext-dcatdonl plugin to function properly, a background process must run at
+least once a day. This background process retrieves the latest versions of the valuelists and saves
+these locally. This process is run by executing the following command once a day via a CRON job for
+example.
+
+.. code-block:: bash
+
+    python /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/task/controlled_vocabulary_updater.py
+
+Ensure that the python script has READ and WRITE access to the following directory and its contents
+
+The extension provides a `.sh` file which executes the above command, this file can easily be added to your
+servers crontab. This file is located in `shell/valuelist_updater.sh`.
+
+.. code-block:: bash
+
+    /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/controlled_vocabularies
+
+The extension can function without the background process running, however this means that the
+valuelists that are used as part of the DCAT-AP-DONL metadata standard will never be updated.

source/installation-plugin.rst

@@ -0,0 +1,31 @@
+Installing the ckanext-dcatdonl plugin
+===================================================================================================
+
+Follow the steps listed below to install and activate the ckanext-dcatdonl extension into CKAN.
+
+1. With your CKAN virtual environment activated:
+
+.. code-block:: bash
+
+    . /usr/lib/ckan/default/bin/activate
+    pip install -e git+https://gitlab.textinfo.nl/opensource/ckanext-dcatdonl.git#egg=ckanext-dcatdonl
+    cd ckanext-dcatdonl
+    pip install -r requirements.txt
+
+2. Edit your CKAN .ini configuration file and add the following
+
+.. code-block:: ini
+
+    ckan.plugins = ... dcatdonl
+
+3. In the same file, add (or change) the following properties to:
+
+.. code-block:: ini
+
+    licenses_group_url = file:///usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/overheid_license.json
+    solr_url = http://{{host}}:8983/solr/ckan
+    ckan.mimetype_guess = None
+
+4. Restart apache2
+
+You have now successfully installed the `ckanext-dcatdonl` plugin

source/installation-requirements.rst

@@ -0,0 +1,35 @@
+Requirements
+===========================================
+
+The plugin was developed with the following versions in mind.
+
+CKAN
+-------------------------------------------
+The plugin functions correctly with these CKAN versions:
+
+.. list-table::
+    :widths: 25 75
+    :header-rows: 1
+
+    * - Version
+      - Reference
+    * - `2.7.3`
+      - http://docs.ckan.org/en/ckan-2.7.3/
+    * - `2.7.4`
+      - http://docs.ckan.org/en/2.7/
+    * - `2.8.0`
+      - http://docs.ckan.org/en/2.8/
+
+It is likely that the plugin functions correctly in earlier and later versions, however only the
+above mentioned CKAN versions have been tested and confirmed to work.
+
+PostgreSQL
+-------------------------------------------
+
+CKAN uses PostgreSQL with version :code:`9.2` or higher.
+
+Python
+-------------------------------------------
+
+CKAN itself, and the ckanext-dcatdonl plugin are written in :code:`Python 2.7.x`. As such, the
+CKAN host must have this version of Python installed.

source/installation-solr.rst

@@ -0,0 +1,39 @@
+Installation of Solr
+===================================================================================================
+
+To install Solr version 7.5.0 (assuming no previous Solr installation):
+
+.. code-block:: bash
+
+    sudo apt-get install openjdk-9-jre-headless
+    cd /opt
+    sudo wget http://www-eu.apache.org/dist/lucene/solr/7.5.0/solr-7.5.0.tgz
+    sudo tar xzf solr-7.5.0.tgz solr-7.5.0/bin/install_solr_service.sh --strip-components=2
+    sudo bash ./install_solr_service.sh solr-7.5.0.tgz
+
+To create the CKAN core into Solr:
+
+.. code-block:: bash
+
+    sudo -u solr /opt/solr/bin/solr create -c ckan
+    sudo rm /var/solr/data/ckan/conf/protwords.txt
+    sudo rm /var/solr/data/ckan/conf/solrconfig.xml
+    sudo rm /var/solr/data/ckan/conf/managed-schema
+    sudo rm /var/solr/data/ckan/conf/stopwords.txt
+    sudo rm /var/solr/data/ckan/conf/synonyms.txt
+    sudo mkdir /var/lib/solr
+    sudo chown solr /var/lib/solr -R
+    cd ~
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/currency.xml /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/elevate.xml /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/protwords.txt /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/schema.xml /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/solrconfig.xml /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/spellings.txt /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/stopwords.txt /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/synonyms.txt /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/synonyms_themes.txt /var/solr/data/ckan/conf
+    sudo ln -s /usr/lib/ckan/default/src/ckanext-dcatdonl/ckanext/dcatdonl/resources/solr/7.4/synonyms_themes_hierarchy.txt /var/solr/data/ckan/conf
+    sudo service solr restart
+
+If your want to use the ckanext-dcatdonl solr optimizations for earlier CKAN versions it is advised to use the files present in the `ckanext/dcatdonl/resources/solr/5.5` directory instead.