From feaf3d339bad976f607558ea516cb0dc4cc79853 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Tue, 5 Jun 2018 16:56:10 +0200 Subject: [PATCH 1/8] use current version for older releases --- doc/conf.py | 36 +++++++++++++++++-- .../install-from-docker-compose.rst | 2 +- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index a1936b6e52c..f9c4a71688f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -183,10 +183,26 @@ def get_status_of_this_version(): return 'unsupported' +def get_current_release_tag(): + ''' Return the name of the tag for the current release + + e.g.: "ckan-2.7.4" + + ''' + release_tags_ = get_release_tags() + + current_tag = "ckan-{}".format(version) + + if release_tags_.__contains__(current_tag): + return current_tag + else: + return 'COULD_NOT_DETECT_TAG_VERSION' + + def get_latest_release_tag(): '''Return the name of the git tag for the latest stable release. - e.g.: "ckan-2.1.1" + e.g.: "ckan-2.7.4" This requires git to be installed. @@ -212,6 +228,19 @@ def get_latest_release_version(): return version +def get_current_release_version(): + '''Return the version number of the current release. + + e.g. "2.1.1" + + ''' + version = get_current_release_tag()[len('ckan-'):] + + # TODO: We could assert here that latest_version matches X.Y.Z. + + return version + + def get_latest_package_name(distro='trusty'): '''Return the filename of the Ubuntu package for the latest stable release. @@ -266,7 +295,8 @@ def write_substitutions_file(**kwargs): f.write('.. |{name}| replace:: {substitution}\n'.format( name=name, substitution=substitution)) - +current_release_tag = get_current_release_tag() +current_release_version = get_current_release_version() latest_release_tag_value = get_latest_release_tag() latest_release_version = get_latest_release_version() latest_minor_version = latest_release_version[:3] @@ -276,7 +306,7 @@ def write_substitutions_file(**kwargs): write_substitutions_file( latest_release_tag=latest_release_tag_value, - latest_release_version=get_latest_release_version(), + latest_release_version=latest_release_version, latest_package_name_precise=get_latest_package_name('precise'), latest_package_name_trusty=get_latest_package_name('trusty'), latest_package_name_xenial=get_latest_package_name('xenial'), diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index dfdbbe4893a..bef21d2f7f5 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -71,7 +71,7 @@ Clone CKAN into a directory of your choice:: This will use the latest CKAN master, which may not be stable enough for production use. To use a stable version, checkout the respective tag, e.g.:: - git checkout tags/ckan-2.6.2 + git checkout tags/|latest_release_tag| ---------------------- 2. Build Docker images From d3ad3bc74573965268539be1710e7f9b6ddfb0c6 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Tue, 5 Jun 2018 22:15:35 +0200 Subject: [PATCH 2/8] Set tag with variable --- doc/maintaining/installing/install-from-docker-compose.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index bef21d2f7f5..9c2fca271ec 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -69,9 +69,11 @@ Clone CKAN into a directory of your choice:: git clone https://github.com/ckan/ckan.git This will use the latest CKAN master, which may not be stable enough for production use. -To use a stable version, checkout the respective tag, e.g.:: +To use a stable version, checkout the respective tag, e.g.: - git checkout tags/|latest_release_tag| + .. parsed-literal:: + + git checkout tags/|latest_release_tag| ---------------------- 2. Build Docker images From 87ab28d0f75732914ca6e5575822d8df504b948d Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Wed, 6 Jun 2018 11:41:22 +0200 Subject: [PATCH 3/8] fix icon --- ckan/templates-bs2/snippets/license.html | 2 +- ckan/templates/snippets/license.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates-bs2/snippets/license.html b/ckan/templates-bs2/snippets/license.html index 364dfa9f32f..85ce763a7d0 100644 --- a/ckan/templates-bs2/snippets/license.html +++ b/ckan/templates-bs2/snippets/license.html @@ -18,7 +18,7 @@ {% block license_wrapper %}
{% block license_title %} -

{{ _('License') }}

+

{{ _('License') }}

{% endblock %} {% block license_content %}

diff --git a/ckan/templates/snippets/license.html b/ckan/templates/snippets/license.html index 364dfa9f32f..85ce763a7d0 100644 --- a/ckan/templates/snippets/license.html +++ b/ckan/templates/snippets/license.html @@ -18,7 +18,7 @@ {% block license_wrapper %}

{% block license_title %} -

{{ _('License') }}

+

{{ _('License') }}

{% endblock %} {% block license_content %}

From af9afa90579f22b2a27520e619f0d84ad3adaf8c Mon Sep 17 00:00:00 2001 From: Goce Mitevski Date: Thu, 7 Jun 2018 10:22:52 +0200 Subject: [PATCH 4/8] Fix issue #4273 --- ckan/public/base/less/masthead.less | 3 --- 1 file changed, 3 deletions(-) diff --git a/ckan/public/base/less/masthead.less b/ckan/public/base/less/masthead.less index 791deb32185..d3c19b4d68d 100644 --- a/ckan/public/base/less/masthead.less +++ b/ckan/public/base/less/masthead.less @@ -119,9 +119,6 @@ .section { float: left; } - input[type="text"] { - border-color: darken(@mastheadBackgroundColor, 5); - } .navigation { &.section { @media (min-width: @screen-sm-min) and (max-width: @screen-sm-max) { From 1b9f5655de08e059ecaea67e916ddb6f8d2acb2c Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 7 Jun 2018 10:43:40 +0200 Subject: [PATCH 5/8] remove storage controller --- ckan/config/routing.py | 4 -- ckan/controllers/storage.py | 98 ----------------------------- ckan/tests/test_coding_standards.py | 1 - 3 files changed, 103 deletions(-) delete mode 100644 ckan/controllers/storage.py diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 60209d13692..432c3822454 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -309,10 +309,6 @@ def make_map(): m.connect('/revision/list', action='list') m.connect('/revision/{id}', action='read') - with SubMapper(map, controller='ckan.controllers.storage:StorageController') as m: - m.connect('storage_file', '/storage/f/{label:.*}', - action='file') - with SubMapper(map, controller='util') as m: m.connect('/i18n/strings_{lang}.js', action='i18n_js_strings') m.connect('/util/redirect', action='redirect') diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py deleted file mode 100644 index 536bef679e2..00000000000 --- a/ckan/controllers/storage.py +++ /dev/null @@ -1,98 +0,0 @@ -# encoding: utf-8 - -''' - -Note: This is the old file store controller for CKAN < 2.2. -If you are looking for how the file uploads work, you should check -`lib/uploader.py` and the `resource_download` method of the package -controller. - -''' -import os -import re - -from ofs import get_impl -from paste.fileapp import FileApp - -from ckan.lib.base import BaseController, request, config, h, abort - - -from logging import getLogger -log = getLogger(__name__) - - -BUCKET = config.get('ckan.storage.bucket', 'default') -key_prefix = config.get('ckan.storage.key_prefix', 'file/') - -_eq_re = re.compile(r"^(.*)(=[0-9]*)$") - - -def create_pairtree_marker(folder): - """ Creates the pairtree marker for tests if it doesn't exist """ - if not folder[:-1] == '/': - folder = folder + '/' - - directory = os.path.dirname(folder) - if not os.path.exists(directory): - os.makedirs(directory) - - target = os.path.join(directory, 'pairtree_version0_1') - if os.path.exists(target): - return - - open(target, 'wb').close() - - -def get_ofs(): - """Return a configured instance of the appropriate OFS driver. - """ - storage_backend = config['ofs.impl'] - kw = {} - for k, v in config.items(): - if not k.startswith('ofs.') or k == 'ofs.impl': - continue - kw[k[4:]] = v - - # Make sure we have created the marker file to avoid pairtree issues - if storage_backend == 'pairtree' and 'storage_dir' in kw: - create_pairtree_marker(kw['storage_dir']) - - ofs = get_impl(storage_backend)(**kw) - return ofs - - -class StorageController(BaseController): - '''Upload to storage backend. - ''' - _ofs_impl = None - - @property - def ofs(self): - if not StorageController._ofs_impl: - StorageController._ofs_impl = get_ofs() - return StorageController._ofs_impl - - def file(self, label): - exists = self.ofs.exists(BUCKET, label) - if not exists: - # handle erroneous trailing slash by redirecting to url w/o slash - if label.endswith('/'): - label = label[:-1] - # This may be best being cached_url until we have moved it into - # permanent storage - file_url = h.url_for('storage_file', label=label) - h.redirect_to(file_url) - else: - abort(404) - - file_url = self.ofs.get_url(BUCKET, label) - if file_url.startswith("file://"): - metadata = self.ofs.get_metadata(BUCKET, label) - filepath = file_url[len("file://"):] - headers = { - # 'Content-Disposition':'attachment; filename="%s"' % label, - 'Content-Type': metadata.get('_format', 'text/plain')} - fapp = FileApp(filepath, headers=None, **headers) - return fapp(request.environ, self.start_response) - else: - h.redirect_to(file_url.encode('ascii', 'ignore')) diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index b8ef315eb31..4f83f9dcd75 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -260,7 +260,6 @@ def find_unprefixed_string_literals(filename): u'ckan/controllers/package.py', u'ckan/controllers/partyline.py', u'ckan/controllers/revision.py', - u'ckan/controllers/storage.py', u'ckan/controllers/tag.py', u'ckan/controllers/user.py', u'ckan/controllers/util.py', From 1ca5becb28d28517d99a57244f1a44079c10e264 Mon Sep 17 00:00:00 2001 From: Konstantin Sivakov Date: Thu, 7 Jun 2018 14:03:33 +0200 Subject: [PATCH 6/8] small fix for #3991 --- doc/maintaining/installing/install-from-docker-compose.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/maintaining/installing/install-from-docker-compose.rst b/doc/maintaining/installing/install-from-docker-compose.rst index 9c2fca271ec..ecf6f915f33 100644 --- a/doc/maintaining/installing/install-from-docker-compose.rst +++ b/doc/maintaining/installing/install-from-docker-compose.rst @@ -175,7 +175,7 @@ a. Create and configure datastore database With running CKAN containers, execute the built-in setup scripts against the ``db`` container:: - docker exec -it db psql -U ckan -f 00_create_datastore.sql + docker exec -it db psql -U ckan -f 00_create_datastore.sh docker exec ckan /usr/local/bin/ckan-paster --plugin=ckan datastore set-permissions -c /etc/ckan/production.ini | docker exec -i db psql -U ckan The first script will create the datastore database and the datastore readonly user in the ``db`` From 716a1ea652d4fcd00ea777825a6c98f6b916eadc Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 7 Jun 2018 15:14:11 +0100 Subject: [PATCH 7/8] Replace HTTPIE examples with CURL examples As per #2293 we replace the httpie examples with the equivalent in curl. It appears that the examples were using http and therefore were getting a 302 to the https url, and as we all know to our pain, POST data doesn't follow redirects and so the examples weren't always working. Have changes some occurrences of http://demo.ckan.org to https://demo.ckan.org --- doc/api/index.rst | 18 +++++++++--------- doc/contributing/documentation.rst | 2 +- doc/maintaining/linked-data-and-rdf.rst | 8 ++++---- doc/maintaining/stats.rst | 2 +- doc/user-guide.rst | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/doc/api/index.rst b/doc/api/index.rst index 23c86f595d2..8f42754441d 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -73,13 +73,13 @@ To call the CKAN API, post a JSON dictionary in an HTTP POST request to one of CKAN's API URLs. The parameters for the API function should be given in the JSON dictionary. CKAN will also return its response in a JSON dictionary. -One way to post a JSON dictionary to a URL is using the command-line HTTP -client `HTTPie `_. For example, to get a list of the names +One way to post a JSON dictionary to a URL is using the command-line +client `Curl `_. For example, to get a list of the names of all the datasets in the ``data-explorer`` group on demo.ckan.org, install -HTTPie and then call the ``group_list`` API function by running this command +curl and then call the ``group_list`` API function by running this command in a terminal:: - http http://demo.ckan.org/api/3/action/group_list + curl http://demo.ckan.org/api/3/action/group_list The response from CKAN will look like this:: @@ -262,16 +262,16 @@ can be configured with the ``apikey_header_name`` option in your CKAN configuration file.) For example, to ask whether or not you're currently following the user -``markw`` on demo.ckan.org using HTTPie, run this command:: +``markw`` on demo.ckan.org using curl, run this command:: - http http://demo.ckan.org/api/3/action/am_following_user id=markw Authorization:XXX + curl -H "Authorization: XXX" https://demo.ckan.org/api/3/action/am_following_user?id=markw (Replacing ``XXX`` with your API key.) Or, to get the list of activities from your user dashboard on demo.ckan.org, run this Python code:: - request = urllib2.Request('http://demo.ckan.org/api/3/action/dashboard_activity_list') + request = urllib2.Request('https://demo.ckan.org/api/3/action/dashboard_activity_list') request.add_header('Authorization', 'XXX') response_dict = json.loads(urllib2.urlopen(request, '{}').read()) @@ -374,9 +374,9 @@ Uploading a new version of a resource file You can use the ``upload`` parameter of the :py:func:`~ckan.logic.action.update.resource_update` function to upload a new version of a resource file. This requires a ``multipart/form-data`` -request, with httpie you can do this using the ``@file.csv``:: +request, with curl you can do this using the ``@file.csv``:: - http --json POST http://demo.ckan.org/api/3/action/resource_update id= upload=@updated_file.csv Authorization: + curl -X POST -H "Content-Type: multipart/form-data" -H "Authorization: XXXX" -F "id=" -F "upload=@updated_file.csv" https://demo.ckan.org/api/3/action/resource_update .. _api-reference: diff --git a/doc/contributing/documentation.rst b/doc/contributing/documentation.rst index b45f45e77c6..bbe6cf16db0 100644 --- a/doc/contributing/documentation.rst +++ b/doc/contributing/documentation.rst @@ -492,7 +492,7 @@ is a nice way to include a list of related links:: :doc:`The DataStore extension ` A CKAN extension for storing data. - CKAN's `demo site `_ + CKAN's `demo site `_ A demo site running the latest CKAN beta version. Seealso boxes are particularly useful when two pages are related, but don't diff --git a/doc/maintaining/linked-data-and-rdf.rst b/doc/maintaining/linked-data-and-rdf.rst index 80676ee9f25..8f52fedef5b 100644 --- a/doc/maintaining/linked-data-and-rdf.rst +++ b/doc/maintaining/linked-data-and-rdf.rst @@ -9,10 +9,10 @@ https://github.com/ckan/ckanext-dcat These features include the RDF serializations of CKAN datasets based on `DCAT`_, that used to be generated using templates hosted on the main CKAN repo, eg: -* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.xml -* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.ttl -* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.n3 -* http://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.jsonld +* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.xml +* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.ttl +* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.n3 +* https://demo.ckan.org/dataset/newcastle-city-council-payments-over-500.jsonld ckanext-dcat offers many more `features `_, including catalog-wide endpoints and harvesters to import RDF data into CKAN. Please check diff --git a/doc/maintaining/stats.rst b/doc/maintaining/stats.rst index fee5f4cfa60..15b76192025 100644 --- a/doc/maintaining/stats.rst +++ b/doc/maintaining/stats.rst @@ -42,4 +42,4 @@ Viewing the Statistics ====================== To view the statistics reported by the stats extension, visit the ``/stats`` -page, for example: http://demo.ckan.org/stats +page, for example: https://demo.ckan.org/stats diff --git a/doc/user-guide.rst b/doc/user-guide.rst index 51ea26d4c43..cb40061736e 100644 --- a/doc/user-guide.rst +++ b/doc/user-guide.rst @@ -140,7 +140,7 @@ Adding a new dataset You may need to be a member of an organization in order to add and edit datsets. See the section :ref:`creating_an_organization` below. On - http://demo.ckan.org, you can add a dataset without being in an organization, + https://demo.ckan.org, you can add a dataset without being in an organization, but dataset features relating to authorization and organizations will not be available. From 37fb5bc9be2d81dcfa2a2ae42bc532b7a2c61e02 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 8 Jun 2018 08:29:27 +0100 Subject: [PATCH 8/8] Fix a missing http->https change --- doc/api/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/index.rst b/doc/api/index.rst index 8f42754441d..1fb3e117089 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -79,7 +79,7 @@ of all the datasets in the ``data-explorer`` group on demo.ckan.org, install curl and then call the ``group_list`` API function by running this command in a terminal:: - curl http://demo.ckan.org/api/3/action/group_list + curl https://demo.ckan.org/api/3/action/group_list The response from CKAN will look like this::