Merge pull request #7772 from ckan/extract-tracking-to-own-extension

Extract tracking to own core extension
ckan · Oct 30, 2023 · 4b80665 · 4b80665
2 parents f886562 + 21f110a
commit 4b80665
Show file tree

Hide file tree

Showing 54 changed files with 1,004 additions and 834 deletions.
diff --git a/.flake8 b/.flake8
@@ -50,7 +50,6 @@ extend-exclude =
                ckan/model/tag.py
                ckan/model/task_status.py
                ckan/model/term_translation.py
-               ckan/model/tracking.py
                ckan/model/user.py
                ckan/model/vocabulary.py
                ckan/authz.py

diff --git a/changes/7772.misc b/changes/7772.misc
@@ -0,0 +1,7 @@
+Tracking feature has been moved to its own core extension. Therefore, `ckan.tracking_enabled`
+configuration option should be changed to adding `tracking` to CKAN's plugins list.
+
+`g.tracking_enabled` attribute no longer exist.
+
+`tracking_summary` info will be returned if the extension is enabled. `include_tracking`
+parameter is no longer required.
diff --git a/ckan/cli/cli.py b/ckan/cli/cli.py
@@ -26,7 +26,6 @@
     profile,
     sass,
     sysadmin,
-    tracking,
     translation,
     user,
     views,
@@ -229,7 +228,6 @@ def ckan():
 ckan.add_command(search_index.search_index)
 ckan.add_command(server.run)
 ckan.add_command(sysadmin.sysadmin)
-ckan.add_command(tracking.tracking)
 ckan.add_command(translation.translation)
 ckan.add_command(user.user)
 ckan.add_command(views.views)

diff --git a/ckan/config/config_declaration.yaml b/ckan/config/config_declaration.yaml
@@ -450,11 +450,6 @@ groups:
         example: 2592000
         description: Controls CKAN static files' cache max age, if we're serving and caching them.
 
-      - key: ckan.tracking_enabled
-        type: bool
-        example: "true"
-        description: This controls if CKAN will track the site usage. For more info, read :ref:`tracking`.
-
       - key: ckan.valid_url_schemes
         type: list
         default:

diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py
@@ -1,14 +1,11 @@
 # encoding: utf-8
 
 """Additional middleware used by the Flask app stack."""
-import hashlib
 from typing import Any
 
-from urllib.parse import unquote, urlparse
+from urllib.parse import urlparse
 
-import sqlalchemy as sa
-
-from ckan.common import CKANConfig, config
+from ckan.common import config
 from ckan.types import CKANApp
 
 
@@ -30,42 +27,6 @@ def __call__(self, environ: Any, start_response: Any):
         return self.app(environ, start_response)
 
 
-class TrackingMiddleware(object):
-
-    def __init__(self, app: CKANApp, config: CKANConfig):
-        self.app = app
-        self.engine = sa.create_engine(config.get('sqlalchemy.url'))
-
-    def __call__(self, environ: Any, start_response: Any) -> Any:
-        path = environ['PATH_INFO']
-        method = environ.get('REQUEST_METHOD')
-        if path == '/_tracking' and method == 'POST':
-            # wsgi.input is a BytesIO object
-            payload = environ['wsgi.input'].read().decode()
-            parts = payload.split('&')
-            data = {}
-            for part in parts:
-                k, v = part.split('=')
-                data[k] = unquote(v)
-            start_response('200 OK', [('Content-Type', 'text/html')])
-            # we want a unique anonomized key for each user so that we do
-            # not count multiple clicks from the same user.
-            key = ''.join([
-                environ['HTTP_USER_AGENT'],
-                environ['REMOTE_ADDR'],
-                environ.get('HTTP_ACCEPT_LANGUAGE', ''),
-                environ.get('HTTP_ACCEPT_ENCODING', ''),
-            ])
-            key = hashlib.md5(key.encode()).hexdigest()
-            # store key/data here
-            sql = '''INSERT INTO tracking_raw
-                     (user_key, url, tracking_type)
-                     VALUES (%s, %s, %s)'''
-            self.engine.execute(sql, key, data.get('url'), data.get('type'))
-            return []
-        return self.app(environ, start_response)
-
-
 class HostHeaderMiddleware(object):
     '''
         Prevent the `Host` header from the incoming request to be used

diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py
@@ -38,8 +38,7 @@
 from ckan.lib import i18n
 from ckan.lib.flask_multistatic import MultiStaticFlask
 from ckan.common import config, g, request, ungettext
-from ckan.config.middleware.common_middleware import (TrackingMiddleware,
-                                                      HostHeaderMiddleware,
+from ckan.config.middleware.common_middleware import (HostHeaderMiddleware,
                                                       RootPathMiddleware)
 import ckan.lib.app_globals as app_globals
 import ckan.lib.plugins as lib_plugins
@@ -299,9 +298,6 @@ def load_user_from_request(request):  # type: ignore
 
     app = I18nMiddleware(app)
 
-    if config.get('ckan.tracking_enabled'):
-        app = TrackingMiddleware(app, config)
-
     # Add a reference to the actual Flask app so it's easier to access
     # type_ignore_reason: custom attribute
     app._wsgi_app = flask_app  # type: ignore

diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py
@@ -53,7 +53,6 @@
     # bool
     'debug': {'default': 'false', 'type' : 'bool'},
     'ckan.debug_supress_header' : {'default': 'false', 'type' : 'bool'},
-    'ckan.tracking_enabled' : {'default': 'false', 'type' : 'bool'},
 
     # int
     'ckan.datasets_per_page': {'default': '20', 'type': 'int'},

diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
@@ -56,7 +56,7 @@
 
 
 from ckan.lib.pagination import Page  # type: ignore # noqa: re-export
-from ckan.common import _, ungettext, g, request, json
+from ckan.common import _, g, request, json
 
 from ckan.lib.webassets_tools import include_asset, render_assets
 from markupsafe import Markup, escape
@@ -1937,23 +1937,6 @@ def debug_inspect(arg: Any) -> Markup:
     return literal('<pre>') + pprint.pformat(arg) + literal('</pre>')
 
 
-@core_helper
-def popular(type_: str,
-            number: int,
-            min: int = 1,
-            title: Optional[str] = None) -> str:
-    ''' display a popular icon. '''
-    if type_ == 'views':
-        title = ungettext('{number} view', '{number} views', number)
-    elif type_ == 'recent views':
-        title = ungettext('{number} recent view', '{number} recent views',
-                          number)
-    elif not title:
-        raise Exception('popular() did not recieve a valid type_ or title')
-    return snippet('snippets/popular.html',
-                   title=title, number=number, min=min)
-
-
 @core_helper
 def groups_available(am_member: bool = False,
                      include_dataset_count: bool = False,
@@ -2178,7 +2161,7 @@ def render_markdown(data: str,
 def format_resource_items(
         items: list[tuple[str, Any]]) -> list[tuple[str, Any]]:
     ''' Take a resource item list and format nicely with blacklisting etc. '''
-    blacklist = ['name', 'description', 'url', 'tracking_summary']
+    blacklist = ['name', 'description', 'url']
     output = []
     # regular expressions for detecting types in strings
     reg_ex_datetime = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{6})?$'

diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
@@ -110,11 +110,6 @@ def index_package(self,
         if pkg_dict is None:
             return
 
-        # tracking summary values will be stale, never store them
-        tracking_summary = pkg_dict.pop('tracking_summary', None)
-        for r in pkg_dict.get('resources', []):
-            r.pop('tracking_summary', None)
-
         # Index validated data-dict
         package_plugin = lib_plugins.lookup_package_plugin(
             pkg_dict.get('type'))
@@ -191,13 +186,6 @@ def index_package(self,
         else:
            pkg_dict['organization'] = None
 
-        # tracking
-        if not tracking_summary:
-            tracking_summary = model.TrackingSummary.get_for_package(
-                pkg_dict['id'])
-        pkg_dict['views_total'] = tracking_summary['total']
-        pkg_dict['views_recent'] = tracking_summary['recent']
-
         resource_fields = [('name', 'res_name'),
                            ('description', 'res_description'),
                            ('format', 'res_format'),

diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
@@ -985,9 +985,6 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS
     :param use_default_schema: use default package schema instead of
         a custom schema defined with an IDatasetForm plugin (default: ``False``)
     :type use_default_schema: bool
-    :param include_tracking: add tracking information to dataset and
-        resources (default: ``False``)
-    :type include_tracking: bool
     :param include_plugin_data: Include the internal plugin data object
         (sysadmin only, optional, default:``False``)
     :type: include_plugin_data: bool
@@ -1021,7 +1018,6 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS
 
     if data_dict.get('use_default_schema', False):
         context['schema'] = ckan.logic.schema.default_show_package_schema()
-    include_tracking = asbool(data_dict.get('include_tracking', False))
 
     package_dict = None
     use_cache = (context.get('use_cache', True))
@@ -1054,16 +1050,6 @@ def package_show(context: Context, data_dict: DataDict) -> ActionResult.PackageS
         )
         package_dict_validated = False
 
-    if include_tracking:
-        # page-view tracking summary data
-        package_dict['tracking_summary'] = (
-            model.TrackingSummary.get_for_package(package_dict['id']))
-
-        for resource_dict in package_dict['resources']:
-            summary =  model.TrackingSummary.get_for_resource(
-                resource_dict['url'])
-            resource_dict['tracking_summary'] = summary
-
     if context.get('for_view'):
         for item in plugins.PluginImplementations(plugins.IPackageController):
             package_dict = item.before_dataset_view(package_dict)
@@ -1098,9 +1084,6 @@ def resource_show(
 
     :param id: the id of the resource
     :type id: string
-    :param include_tracking: add tracking information to dataset and
-        resources (default: ``False``)
-    :type include_tracking: bool
 
     :rtype: dictionary
 
@@ -1116,9 +1099,8 @@ def resource_show(
     _check_access('resource_show', resource_context, data_dict)
 
     pkg_dict = logic.get_action('package_show')(
-        context.copy(),
-        {'id': resource.package.id,
-        'include_tracking': asbool(data_dict.get('include_tracking', False))})
+        context.copy(), {'id': resource.package.id}
+        )
 
     for resource_dict in pkg_dict['resources']:
         if resource_dict['id'] == id:

diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py
@@ -58,7 +58,6 @@ def default_resource_schema(
         'created': [ignore_missing, isodate],
         'last_modified': [ignore_missing, isodate],
         'cache_last_updated': [ignore_missing, isodate],
-        'tracking_summary': [ignore_missing],
         'datastore_active': [ignore_missing],
         '__extras': [ignore_missing, extras_valid_json, keep_extras],
     }
@@ -261,7 +260,6 @@ def default_show_package_schema(keep_extras: Validator,
     schema['organization'] = []
     schema['owner_org'] = []
     schema['private'] = []
-    schema['tracking_summary'] = [ignore_missing]
     schema['license_title'] = []
 
     return schema

diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py
@@ -72,12 +72,6 @@
     ResourceView,
     resource_view_table,
 )
-from ckan.model.tracking import (
-    tracking_summary_table,
-    TrackingSummary,
-    tracking_raw_table
-)
-
 from ckan.model.package_relationship import (
     PackageRelationship,
     package_relationship_table,
@@ -132,7 +126,6 @@
     "GroupExtra", "group_extra_table", "PackageExtra", "package_extra_table",
     "Resource", "DictProxy", "resource_table",
     "ResourceView", "resource_view_table",
-    "tracking_summary_table", "TrackingSummary", "tracking_raw_table",
     "PackageRelationship", "package_relationship_table",
     "TaskStatus", "task_status_table",
     "Vocabulary", "VOCABULARY_NAME_MAX_LENGTH", "VOCABULARY_NAME_MIN_LENGTH",

diff --git a/ckan/model/tracking.py b/ckan/model/tracking.py
diff --git a/ckan/public/base/css/main-rtl.css b/ckan/public/base/css/main-rtl.css
@@ -12459,10 +12459,6 @@ select[data-module=autocomplete] {
   background-color: #6e6e6e;
 }
 
-.popular {
-  color: #f29c2b;
-}
-
 .resource-list {
   margin: 0;
   list-style: none;

diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css
@@ -12459,10 +12459,6 @@ select[data-module=autocomplete] {
   background-color: #6e6e6e;
 }
 
-.popular {
-  color: #f29c2b;
-}
-
 .resource-list {
   margin: 0;
   list-style: none;

diff --git a/ckan/public/base/javascript/resource.config b/ckan/public/base/javascript/resource.config
@@ -1,7 +1,5 @@
 [main]
 
-dont_bundle = tracking.js
-
 [depends]
 
 main = vendor/vendor