AV-1986: CKAN 2.10 + Python 3.10

.github/workflows/check.yml

@@ -98,7 +98,7 @@ jobs:
         working-directory: ./opendata-assets
 
       - name: run stylelint
-        run: npx stylelint "src/less" -f github
+        run: npx stylelint "src/scss" -f github
         working-directory: ./opendata-assets
 
   slack:

.stylelintignore

@@ -1 +1 @@
-modules/ytp-assets-common/src/less/ckan/upstream_ckan
+modules/ytp-assets-common/src/scss/ckan/upstream_ckan

ckan/Dockerfile

@@ -8,11 +8,14 @@ ARG SECRET_NPMRC
 ##################
 ### Build CKAN ###
 ##################
-FROM public.ecr.aws/docker/library/python:3.9.16-alpine3.17 as base_ckan_build
+FROM public.ecr.aws/docker/library/python:3.10.9-alpine3.17 as base_ckan_build
+
+# Used by Github Actions to tag the image with
+ENV IMAGE_TAG=2.10.0
 
 # Set CKAN version to build
 ENV GIT_URL=https://github.com/ckan/ckan.git
-ENV GIT_BRANCH=ckan-2.9.9
+ENV GIT_BRANCH=ckan-2.10.0
 
 # Set src dirs
 ENV SRC_DIR=/srv/app/src
@@ -47,7 +50,7 @@ RUN apk add --no-cache \
         libxml2-dev \
         libxslt-dev
 
-# Create the src and pip cache directory
+# Create the src directory
 RUN mkdir -p ${SRC_DIR}
 
 # Install pip
@@ -67,7 +70,7 @@ RUN pip wheel --wheel-dir=/wheels uWSGI==2.0.20 gevent==21.12.0 greenlet==1.1.3
 ###########################
 ### Default-Extensions ####
 ###########################
-FROM public.ecr.aws/docker/library/python:3.9.16-alpine3.17 as base_extbuild
+FROM public.ecr.aws/docker/library/python:3.10.9-alpine3.17 as base_extbuild
 
 # Set src dirs
 ENV SRC_DIR=/srv/app/src
@@ -99,7 +102,7 @@ RUN pip wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#e
 ############
 ### MAIN ###
 ############
-FROM public.ecr.aws/docker/library/python:3.9.16-alpine3.17 as base_ckan
+FROM public.ecr.aws/docker/library/python:3.10.9-alpine3.17 as base_ckan
 
 ENV APP_DIR=/srv/app
 ENV SRC_DIR=/srv/app/src
@@ -266,7 +269,9 @@ RUN cd ${SRC_DIR}/ckan && \
     patch --strip=1 --input=patches/json_serializable_lazyjsonobject.patch && \
     patch --strip=1 --input=patches/implement_is_required_for_image_upload.patch && \
     patch --strip=1 --input=patches/add_drafts_to_search.patch && \
-    patch --strip=1 --input=patches/add_prefix_to_werkzeug.patch
+    patch --strip=1 --input=patches/fix_license_dictization.patch && \
+    patch --strip=1 --input=patches/fix_truncate_helper_annotations.patch
+
 
 # install crontab
 RUN chmod +x ${CRON_DIR}/scripts/*.sh && \
@@ -300,7 +305,7 @@ ENTRYPOINT ["/srv/app/scripts/entrypoint_ckan.sh"]
 #
 # Modules build (for production)
 #
-FROM public.ecr.aws/docker/library/python:3.9.16-alpine3.17 as modules_build
+FROM public.ecr.aws/docker/library/python:3.10.9-alpine3.17 as modules_build
 
 # install required packages
 RUN apk add curl nodejs libjpeg

ckan/ckanext/ckanext-advancedsearch/ckanext/advancedsearch/controller.py

@@ -2,9 +2,7 @@
 import logging
 import math
 
-from ckan.common import c, request
-from ckan.lib.base import render
-from ckan.logic import get_action
+from ckan.plugins import toolkit
 from flask import Blueprint
 
 from .helpers import advancedsearch_schema, field_options, query_helper
@@ -28,13 +26,13 @@ def search():
     context = {
         'model': model,
         'session': model.Session,
-        'user': c.user
+        'user': toolkit.g.user
     }
 
     # On initial page load there is no page parameter so display the first page
     # On possible page navigations use the page parameter to move to the next page
     # NOTE: this works also with a GET request but the POST filters will not be submitted so all datasets will be returned
-    page = int(request.params.get('page', 1))
+    page = int(toolkit.request.params.get('page', 1))
     # Limit amount of results returned
     limit = 20
     search_query_filters = []
@@ -51,11 +49,11 @@ def search():
         # Make a list of field options
         options[key] = field_options(val)
 
-    if request.method == 'POST':
+    if toolkit.request.method == 'POST':
         # Use the field labelled as the main_query to build the value for q
         # TODO: Handle no main_query_field provided
         main_query_helper = query_helper(schema['input_fields'].get(main_query_field))
-        q = main_query_helper(main_query_field, request.form, schema['input_fields'], context)
+        q = main_query_helper(main_query_field, toolkit.request.form, schema['input_fields'], context)
 
         # Iterate through all fields in schema except the main_query_field
         # and process every field with the provided query_helper
@@ -67,13 +65,13 @@ def search():
             query_helper_function = query_helper(val)
             # TODO: handle no query_helper
             if query_helper_function:
-                res = query_helper_function(key, request.form, schema['input_fields'], context)
+                res = query_helper_function(key, toolkit.request.form, schema['input_fields'], context)
                 if res:
                     search_query_filters.append(res)
 
-    sort_string = request.form.get('sort', 'metadata_created desc')
+    sort_string = toolkit.request.form.get('sort', 'metadata_created desc')
     # if an actual sort parameter is provided, use that for selection in the template
-    sorting_selection = request.form.get('sort')
+    sorting_selection = toolkit.request.form.get('sort')
 
     data_dict = {
         'q': q,
@@ -93,14 +91,16 @@ def search():
         # Outputs: (filter:value) AND (another_filter:another_value)
         data_dict['fq'] = '(' + ') AND ('.join(search_query_filters) + ')'
 
-    query = get_action('package_search')(context, data_dict)
+    log.info(f'q = {data_dict["q"]}, fq = {data_dict["fq"]}')
+    query = toolkit.get_action('package_search')(context, data_dict)
 
     json_query = json.dumps(
-        {k: v for k, v in list(params_to_dict(request.form).items()) if k != 'page' and isinstance(v, list) and len(v[0]) > 0}
+        {k: v for k, v in list(params_to_dict(toolkit.request.form).items())
+              if k != 'page' and isinstance(v, list)and len(v[0]) > 0}
     )
 
     filters = {
-        k: v for k, v in list(params_to_dict(request.form).items()) if k != 'search_target' and k != 'search_query'
+        k: v for k, v in list(params_to_dict(toolkit.request.form).items()) if k != 'search_target' and k != 'search_query'
         and k != 'page' and k != 'released-before' and k != 'released-after' and k != 'updated-before'
         and k != 'updated-after' and k != 'sort' and isinstance(v, list) and len(v[0]) > 0
     }
@@ -117,23 +117,23 @@ def search():
                     options_list.append(x)
             filters[key] = options_list
 
-    c.advanced_search = {
+    toolkit.g.advanced_search = {
         "item_count": query['count'],
         # Round values up to get total amount of pages
         "total_pages": int(math.ceil(float(query['count']) / float(limit))),
         "collection": query['results'],
         # Return query parameters to the UI so that it can populate the fields with the previous query values
         # NOTE: Can this cause security issues? Returning POST request params back to the client
-        "last_query": params_to_dict(request.form),
+        "last_query": params_to_dict(toolkit.request.form),
         "json_query": json_query,
         "filters": filters,
         "sort_string": sort_string,
         "field_options": options,
         "sorting_selection": sorting_selection
         }
-    c.advanced_search['last_query']['page'] = page
+    toolkit.g.advanced_search['last_query']['page'] = page
 
-    return render('advanced_search/index.html')
+    return toolkit.render('advanced_search/index.html')
 
 
 def params_to_dict(params):

ckan/ckanext/ckanext-advancedsearch/ckanext/advancedsearch/templates/advanced_search/index.html

@@ -20,14 +20,13 @@
   <div id="content" class="container advanced-search-container">
     {% block main_content %}
       {% block flash %}
+        {% set flash_messages = h.get_flashed_messages(with_categories=true) %}
         <div class="flash-messages">
-          {% block flash_inner %}
-            {% for message in h.flash.pop_messages() | list %}
-              <div class="alert alert-warning fade in {{ message.category }}">
-                {{ h.literal(message) }}
-              </div>
-            {% endfor %}
-          {% endblock %}
+          {% for category, message in flash_messages %}
+            <div class="m-0 alert alert-warning fade in {{ message.category }}">
+              {{ h.literal(message) }}
+            </div>
+          {% endfor %}
         </div>
       {% endblock %}
 

ckan/ckanext/ckanext-sixodp_showcase/ckanext/sixodp_showcase/logic/schema.py

@@ -1,17 +1,17 @@
 # -*- coding: utf-8 -*-
 
-import six
 from ckan.lib.navl.validators import (not_empty)
 
 from ckanext.sixodp_showcase.logic.validators import (convert_package_name_or_id_to_id_for_type_apiset)
 from ckanext.showcase.logic.validators import (convert_package_name_or_id_to_id_for_type_showcase)
+from ckantoolkit import unicode_safe
 
 
 def showcase_apiset_association_create_schema():
     schema = {
-        'package_id': [not_empty, six.text_type,
+        'package_id': [not_empty, unicode_safe,
                        convert_package_name_or_id_to_id_for_type_apiset],
-        'showcase_id': [not_empty, six.text_type,
+        'showcase_id': [not_empty, unicode_safe,
                         convert_package_name_or_id_to_id_for_type_showcase]
     }
     return schema
@@ -23,15 +23,15 @@ def showcase_apiset_association_delete_schema():
 
 def showcase_apiset_list_schema():
     schema = {
-        'showcase_id': [not_empty, six.text_type,
+        'showcase_id': [not_empty, unicode_safe,
                         convert_package_name_or_id_to_id_for_type_showcase]
     }
     return schema
 
 
 def apiset_showcase_list_schema():
     schema = {
-        'package_id': [not_empty, six.text_type,
+        'package_id': [not_empty, unicode_safe,
                        convert_package_name_or_id_to_id_for_type_showcase]
     }
     return schema

ckan/ckanext/ckanext-sixodp_showcase/ckanext/sixodp_showcase/plugin.py

@@ -180,7 +180,7 @@ def _add_to_pkg_dict(self, context, pkg_dict):
         return pkg_dict
 
     # IPackageController
-    def after_show(self, context, data_dict):
+    def after_dataset_show(self, context, data_dict):
         if context.get('for_edit') is not True:
             if data_dict.get('notifier', None) is not None:
                 data_dict.pop('notifier')
@@ -189,7 +189,7 @@ def after_show(self, context, data_dict):
 
         return self._add_to_pkg_dict(context, data_dict)
 
-    def before_index(self, data_dict):
+    def before_dataset_index(self, data_dict):
         if data_dict.get('platform'):
             data_dict['vocab_platform'] = [tag for tag in data_dict['platform'].split(',')]
 
@@ -208,7 +208,7 @@ def before_index(self, data_dict):
 
         return data_dict
 
-    def after_search(self, search_results, search_params):
+    def after_dataset_search(self, search_results, search_params):
         if(search_results['search_facets'].get('groups')):
             context = {'for_view': True, 'with_private': False}
             data_dict = {
@@ -232,4 +232,4 @@ def after_search(self, search_results, search_params):
     # IClick
 
     def get_commands(self):
-        return cli.get_commands()
+        return cli.get_commands()

ckan/ckanext/ckanext-sixodp_showcase/ckanext/sixodp_showcase/templates/showcase/add_datasets.html

@@ -0,0 +1,88 @@
+{% extends 'showcase/edit_base.html' %}
+
+{% block subtitle %}{{ _('Showcases - Add datasets') }}{% endblock %}
+
+{% block wrapper_class %} ckanext-showcase-edit-wrapper{% endblock %}
+
+{% block ckanext_showcase_edit_span %}span12{% endblock %}
+
+{% block primary_content_inner %}
+  <section class="module">
+    <div class="module-content">
+      {% block form %}
+        {% set facets = {
+          'fields': c.fields_grouped,
+          'search': c.search_facets,
+          'titles': c.facet_titles,
+          'translated_fields': c.translated_fields,
+          'remove_field': c.remove_field }
+        %}
+        {% set sorting = [
+          (_('Relevance'), 'score desc, metadata_created desc'),
+          (_('Name Ascending'), 'title_string asc'),
+          (_('Name Descending'), 'title_string desc'),
+          (_('Last Modified'), 'metadata_modified desc'),
+          (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false) ]
+        %}
+        {% snippet 'snippets/search_form.html', type='dataset', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, count=c.page.item_count, facets=facets, show_empty=request.params, error=c.query_error, fields=c.fields %}
+      {% endblock %}
+       <h3 class="page-heading">
+        {% block page_heading %}
+            {{ _('Datasets available to add to this showcase') }}
+        {% endblock %}
+      </h3>
+      {% block package_search_results_list %}
+        {% if c.page.items %}
+          <form method="POST" data-module="basic-form">
+            {#{% block errors %}{{ form.errors(error_summary) }}{% endblock %}#}
+            <table class="table table-bordered table-header table-hover table-bulk-edit table-edit-hover" data-module="table-selectable-rows">
+              <col width="8">
+              <col width="120">
+              <thead>
+                <tr>
+                  <th></th>
+                  <th class="table-actions">
+                    <div class="btn-group">
+                      <button name="bulk_action.showcase_add" value="add" class="btn" type="submit">
+                        <i class="fa fa-plus icon-plus"></i>
+                        {{ _('Add to Showcase') }}
+                      </button>
+                    </div>
+                  </th>
+                </tr>
+              </thead>
+              <tbody>
+                {% for package in c.page.items %}
+                  {% set truncate = truncate or 180 %}
+                  {% set truncate_title = truncate_title or 80 %}
+                  {% set title = package.title or package.name %}
+                  {% set notes = h.markdown_extract(package.notes, extract_length=truncate) %}
+                  <tr>
+                    <td>
+                      <input type="checkbox" name="dataset_{{ package.id }}">
+                    </td>
+                    <td class="context">
+                      <h3 class="dataset-heading">
+                        {{ h.link_to(h.truncate(title, truncate_title), h.url_for(package.type ~ '.read', id=package.name)) }}
+                      </h3>
+                      {% if notes %}
+                        <p>{{ notes|urlize }}</p>
+                      {% endif %}
+                    </td>
+                  </tr>
+                {% endfor %}
+              </tbody>
+            </table>
+          </form>
+        {% else %}
+          <p class="empty">{{ _('No datasets could be found') }}</p>
+        {% endif %}
+      {% endblock %}
+    </div>
+
+    {% block page_pagination %}
+      {{ c.page.pager(q=c.q) }}
+    {% endblock %}
+  </section>
+
+{% endblock %}

ckan/ckanext/ckanext-sixodp_showcase/ckanext/sixodp_showcase/templates/sixodp_showcase/package_form.html

@@ -6,7 +6,7 @@
 {% block form_actions %}
   <div class="form-actions">
     {% block delete_button %}
-      {% if form_style == 'edit' and h.check_access('ckanext_showcase_delete', {'id': data.id}) and not data.state == 'deleted' %}
+      {% if h.check_access('ckanext_showcase_delete', {'id': data.id}) and not data.state == 'deleted' %}
       <a class="btn suomifi-button-secondary" href="{% url_for 'sixodp_showcase.delete', id=data.id %}" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this showcase?') }}">{% block delete_button_text %}<i class="far fa-trash"></i>{{ _('Delete') }}{% endblock %}</a>
       {% endif %}
       <a class="btn suomifi-button-secondary-noborder" href="{% url_for 'sixodp_showcase.search' %}">{{_("Abort")}}</a>

ckan/ckanext/ckanext-sixodp_showcase/ckanext/sixodp_showcase/templates/sixodp_showcase/snippets/showcase_search_input.html

@@ -13,7 +13,7 @@
             (_('Relevance'), 'score desc, metadata_created desc'),
             (_('Newest first'), 'metadata_created desc'),
             (_('Oldest first'), 'metadata_created asc'),
-            (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false)
+            (_('Popular'), 'views_recent desc') if g.tracking_enabled else (false, false),
             (_('Name Ascending'), 'title_string asc'),
             (_('Name Descending'), 'title_string desc'),
             (_('Last Modified'), 'metadata_modified desc')]