From b670bbfe0dc05b831ef8a4798f44784a4345ac7d Mon Sep 17 00:00:00 2001 From: Carl Lange Date: Mon, 16 May 2016 10:11:43 +0100 Subject: [PATCH 01/18] Add a submit_all command to datapusher. this allows you to add every resource of every package to the datastore. this is useful if you're setting up datastore for a a ckan that's already got datasets. Also renames the existing submit_all function to resubmit_all, because it would not act on resources that had not already been submitted to datastore. --- ckanext/datapusher/cli.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ckanext/datapusher/cli.py b/ckanext/datapusher/cli.py index 7db5bc81711..2328b051f14 100644 --- a/ckanext/datapusher/cli.py +++ b/ckanext/datapusher/cli.py @@ -14,6 +14,9 @@ class DatapusherCommand(cli.CkanCommand): ignoring if their files haven't changed. submit - Submits all resources from the package identified by pkgname (either the short name or ID). + submit_all - Submit every package to the datastore. + This is useful if you're setting up datastore + for a ckan that already has datasets. ''' summary = __doc__.split('\n')[0] @@ -24,7 +27,12 @@ def command(self): self._confirm_or_abort() self._load_config() - self._submit_all() + self._resubmit_all() + elif self.args and self.args[0] == 'submit_all': + self._confirm_or_abort() + + self._load_config() + self._submit_all_packages() elif self.args and self.args[0] == 'submit': self._confirm_or_abort() @@ -49,10 +57,18 @@ def _confirm_or_abort(self): print "Aborting..." sys.exit(0) - def _submit_all(self): + def _resubmit_all(self): resources_ids = datastore_db.get_all_resources_ids_in_datastore() self._submit(resource_ids) + def _submit_all_packages(self): + # submit every package + # for each package in the package list, submit each resource w/ _submit_package + import ckan.model as model + package_list = p.toolkit.get_action('package_list') + for p_id in package_list({'model': model, 'ignore_auth': True}, {}): + self._submit_package(p_id) + def _submit_package(self, pkg_id): import ckan.model as model From d9f3a5036a99c5e447ad93977e99ba745e2868c3 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Wed, 15 Jun 2016 17:19:58 +0100 Subject: [PATCH 02/18] [#3116] Refactor middleware module. It was starting to become unmanageably large with a lot of separate concerns in one place. - Separate the flask and pylons app code into separate modules. - Separate the common middleware code into a separate module. --- ckan/config/middleware.py | 595 -------------------- ckan/config/middleware/__init__.py | 132 +++++ ckan/config/middleware/common_middleware.py | 216 +++++++ ckan/config/middleware/flask_app.py | 81 +++ ckan/config/middleware/pylons_app.py | 218 +++++++ ckan/tests/config/test_middleware.py | 3 +- 6 files changed, 649 insertions(+), 596 deletions(-) delete mode 100644 ckan/config/middleware.py create mode 100644 ckan/config/middleware/__init__.py create mode 100644 ckan/config/middleware/common_middleware.py create mode 100644 ckan/config/middleware/flask_app.py create mode 100644 ckan/config/middleware/pylons_app.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py deleted file mode 100644 index c9898fc11ae..00000000000 --- a/ckan/config/middleware.py +++ /dev/null @@ -1,595 +0,0 @@ -# encoding: utf-8 - -"""Pylons middleware initialization""" -import urllib -import urllib2 -import logging -import json -import hashlib -import os - -import sqlalchemy as sa -from beaker.middleware import CacheMiddleware, SessionMiddleware -from paste.cascade import Cascade -from paste.registry import RegistryManager -from paste.urlparser import StaticURLParser -from paste.deploy.converters import asbool -from pylons import config -from pylons.middleware import ErrorHandler, StatusCodeRedirect -from pylons.wsgiapp import PylonsApp -from routes.middleware import RoutesMiddleware -from repoze.who.config import WhoConfig -from repoze.who.middleware import PluggableAuthenticationMiddleware -from fanstatic import Fanstatic - -from wsgi_party import WSGIParty, HighAndDry -from flask import Flask -from flask import abort as flask_abort -from flask import request as flask_request -from flask import _request_ctx_stack -from werkzeug.exceptions import HTTPException -from werkzeug.test import create_environ, run_wsgi_app - -from ckan.plugins import PluginImplementations -from ckan.plugins.interfaces import IMiddleware -from ckan.lib.i18n import get_locales_from_config -import ckan.lib.uploader as uploader - -from ckan.config.environment import load_environment -import ckan.lib.app_globals as app_globals - -log = logging.getLogger(__name__) - - -def make_app(conf, full_stack=True, static_files=True, **app_conf): - - # :::TODO::: like the flask app, make the pylons app respond to invites at - # /__invite__/, and handle can_handle_request requests. - - pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) - flask_app = make_flask_stack(conf) - - app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, 'flask_app': flask_app}) - - return app - - -def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): - """Create a Pylons WSGI application and return it - - ``conf`` - The inherited configuration for this application. Normally from - the [DEFAULT] section of the Paste ini file. - - ``full_stack`` - Whether this application provides a full WSGI stack (by default, - meaning it handles its own exceptions and errors). Disable - full_stack when this application is "managed" by another WSGI - middleware. - - ``static_files`` - Whether this application serves its own static files; disable - when another web server is responsible for serving them. - - ``app_conf`` - The application's local configuration. Normally specified in - the [app:] section of the Paste ini file (where - defaults to main). - - """ - # Configure the Pylons environment - load_environment(conf, app_conf) - - # The Pylons WSGI app - app = PylonsApp() - # set pylons globals - app_globals.reset() - - for plugin in PluginImplementations(IMiddleware): - app = plugin.make_middleware(app, config) - - app = RootPathMiddleware(app, config) - - # Routing/Session/Cache Middleware - app = RoutesMiddleware(app, config['routes.map']) - # we want to be able to retrieve the routes middleware to be able to update - # the mapper. We store it in the pylons config to allow this. - config['routes.middleware'] = app - app = SessionMiddleware(app, config) - app = CacheMiddleware(app, config) - - # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) - # app = QueueLogMiddleware(app) - if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): - app = execute_on_completion(app, config, cleanup_pylons_response_string) - - # Fanstatic - if asbool(config.get('debug', False)): - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': True, - 'minified': False, - 'bottom': True, - 'bundle': False, - } - else: - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': False, - 'minified': True, - 'bottom': True, - 'bundle': True, - } - app = Fanstatic(app, **fanstatic_config) - - for plugin in PluginImplementations(IMiddleware): - try: - app = plugin.make_error_log_middleware(app, config) - except AttributeError: - log.critical('Middleware class {0} is missing the method' - 'make_error_log_middleware.'.format(plugin.__class__.__name__)) - - if asbool(full_stack): - # Handle Python exceptions - app = ErrorHandler(app, conf, **config['pylons.errorware']) - - # Display error documents for 400, 403, 404 status codes (and - # 500 when debug is disabled) - if asbool(config['debug']): - app = StatusCodeRedirect(app, [400, 403, 404]) - else: - app = StatusCodeRedirect(app, [400, 403, 404, 500]) - - # Initialize repoze.who - who_parser = WhoConfig(conf['here']) - who_parser.parse(open(app_conf['who.config_file'])) - - app = PluggableAuthenticationMiddleware( - app, - who_parser.identifiers, - who_parser.authenticators, - who_parser.challengers, - who_parser.mdproviders, - who_parser.request_classifier, - who_parser.challenge_decider, - logging.getLogger('repoze.who'), - logging.WARN, # ignored - who_parser.remote_user_key - ) - - # Establish the Registry for this application - app = RegistryManager(app) - - app = I18nMiddleware(app, config) - - if asbool(static_files): - # Serve static files - static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ - else int(config.get('ckan.static_max_age', 3600)) - - static_app = StaticURLParser(config['pylons.paths']['static_files'], - cache_max_age=static_max_age) - static_parsers = [static_app, app] - - storage_directory = uploader.get_storage_path() - if storage_directory: - path = os.path.join(storage_directory, 'storage') - try: - os.makedirs(path) - except OSError, e: - # errno 17 is file already exists - if e.errno != 17: - raise - - storage_app = StaticURLParser(path, cache_max_age=static_max_age) - static_parsers.insert(0, storage_app) - - # Configurable extra static file paths - extra_static_parsers = [] - for public_path in config.get('extra_public_paths', '').split(','): - if public_path.strip(): - extra_static_parsers.append( - StaticURLParser(public_path.strip(), - cache_max_age=static_max_age) - ) - app = Cascade(extra_static_parsers + static_parsers) - - # Page cache - if asbool(config.get('ckan.page_cache_enabled')): - app = PageCacheMiddleware(app, config) - - # Tracking - if asbool(config.get('ckan.tracking_enabled', 'false')): - app = TrackingMiddleware(app, config) - - return app - - -def make_flask_stack(conf): - """ This has to pass the flask app through all the same middleware that - Pylons used """ - - app = CKANFlask(__name__) - - @app.route('/hello', methods=['GET']) - def hello_world(): - return 'Hello World, this is served by Flask' - - @app.route('/hello', methods=['POST']) - def hello_world_post(): - return 'Hello World, this was posted to Flask' - - return app - - -class CKANFlask(Flask): - - '''Extend the Flask class with a special view to join the 'partyline' - established by AskAppDispatcherMiddleware. - - Also provide a 'can_handle_request' method. - ''' - - def __init__(self, import_name, *args, **kwargs): - super(CKANFlask, self).__init__(import_name, *args, **kwargs) - self.add_url_rule('/__invite__/', endpoint='partyline', - view_func=self.join_party) - self.partyline = None - self.partyline_connected = False - self.invitation_context = None - self.app_name = None # A label for the app handling this request - # (this app). - - def join_party(self, request=flask_request): - # Bootstrap, turn the view function into a 404 after registering. - if self.partyline_connected: - # This route does not exist at the HTTP level. - flask_abort(404) - self.invitation_context = _request_ctx_stack.top - self.partyline = request.environ.get(WSGIParty.partyline_key) - self.app_name = request.environ.get('partyline_handling_app') - self.partyline.connect('can_handle_request', self.can_handle_request) - self.partyline_connected = True - return 'ok' - - def can_handle_request(self, environ): - ''' - Decides whether it can handle a request with the Flask app by - matching the request environ against the route mapper - - Returns (True, 'flask_app') if this is the case. - ''' - - # TODO: identify matching urls as core or extension. This will depend - # on how we setup routing in Flask - - urls = self.url_map.bind_to_environ(environ) - try: - endpoint, args = urls.match() - log.debug('Flask route match, endpoint: {0}, args: {1}'.format( - endpoint, args)) - return (True, self.app_name) - except HTTPException: - raise HighAndDry() - - -class AskAppDispatcherMiddleware(WSGIParty): - - ''' - Establish a 'partyline' to each provided app. Select which app to call - by asking each if they can handle the requested path at PATH_INFO. - - Used to help transition from Pylons to Flask, and should be removed once - Pylons has been deprecated and all app requests are handled by Flask. - - Each app should handle a call to 'can_handle_request(environ)', responding - with a tuple: - (, , []) - where: - `bool` is True if the app can handle the payload url, - `app` is the wsgi app returning the answer - `origin` is an optional string to determine where in the app the url - will be handled, e.g. 'core' or 'extension'. - - Order of precedence if more than one app can handle a url: - Flask Extension > Pylons Extension > Flask Core > Pylons Core - ''' - - def __init__(self, apps=None, invites=(), ignore_missing_services=False): - # Dict of apps managed by this middleware {: , ...} - self.apps = apps or {} - - # A dict of service name => handler mappings. - self.handlers = {} - - # If True, suppress :class:`NoSuchServiceName` errors. Default: False. - self.ignore_missing_services = ignore_missing_services - - self.send_invitations(apps) - - def send_invitations(self, apps): - '''Call each app at the invite route to establish a partyline. Called - on init.''' - PATH = '/__invite__/' - for app_name, app in apps.items(): - environ = create_environ(path=PATH) - environ[self.partyline_key] = self.operator_class(self) - # A reference to the handling app. Used to id the app when - # responding to a handling request. - environ['partyline_handling_app'] = app_name - run_wsgi_app(app, environ) - - def __call__(self, environ, start_response): - '''Determine which app to call by asking each app if it can handle the - url and method defined on the eviron''' - # :::TODO::: Enforce order of precedence for dispatching to apps here. - - app_name = 'pylons_app' # currently defaulting to pylons app - answers = self.ask_around('can_handle_request', environ) - log.debug('Route support answers for {0} {1}: {2}'.format( - environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), - answers)) - available_handlers = [] - for answer in answers: - if len(answer) == 2: - can_handle, asked_app = answer - origin = 'core' - else: - can_handle, asked_app, origin = answer - if can_handle: - available_handlers.append('{0}_{1}'.format(asked_app, origin)) - - # Enforce order of precedence: - # Flask Extension > Pylons Extension > Flask Core > Pylons Core - if available_handlers: - if 'flask_app_extension' in available_handlers: - app_name = 'flask_app' - elif 'pylons_app_extension' in available_handlers: - app_name = 'pylons_app' - elif 'flask_app_core' in available_handlers: - app_name = 'flask_app' - - log.debug('Serving request via {0} app'.format(app_name)) - environ['ckan.app'] = app_name - return self.apps[app_name](environ, start_response) - - -class RootPathMiddleware(object): - ''' - Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url - config. The routes package uses the SCRIPT_NAME variable and appends to the - path and ckan addes the root url causing a duplication of the root path. - - This is a middleware to ensure that even redirects use this logic. - ''' - def __init__(self, app, config): - self.app = app - - def __call__(self, environ, start_response): - # Prevents the variable interfering with the root_path logic - if 'SCRIPT_NAME' in environ: - environ['SCRIPT_NAME'] = '' - - return self.app(environ, start_response) - - -class I18nMiddleware(object): - """I18n Middleware selects the language based on the url - eg /fr/home is French""" - def __init__(self, app, config): - self.app = app - self.default_locale = config.get('ckan.locale_default', 'en') - self.local_list = get_locales_from_config() - - def __call__(self, environ, start_response): - # strip the language selector from the requested url - # and set environ variables for the language selected - # CKAN_LANG is the language code eg en, fr - # CKAN_LANG_IS_DEFAULT is set to True or False - # CKAN_CURRENT_URL is set to the current application url - - # We only update once for a request so we can keep - # the language and original url which helps with 404 pages etc - if 'CKAN_LANG' not in environ: - path_parts = environ['PATH_INFO'].split('/') - if len(path_parts) > 1 and path_parts[1] in self.local_list: - environ['CKAN_LANG'] = path_parts[1] - environ['CKAN_LANG_IS_DEFAULT'] = False - # rewrite url - if len(path_parts) > 2: - environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) - else: - environ['PATH_INFO'] = '/' - else: - environ['CKAN_LANG'] = self.default_locale - environ['CKAN_LANG_IS_DEFAULT'] = True - - # Current application url - path_info = environ['PATH_INFO'] - # sort out weird encodings - path_info = '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) - - qs = environ.get('QUERY_STRING') - - if qs: - # sort out weird encodings - qs = urllib.quote(qs, '') - environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) - else: - environ['CKAN_CURRENT_URL'] = path_info - - return self.app(environ, start_response) - - -class PageCacheMiddleware(object): - ''' A simple page cache that can store and serve pages. It uses - Redis as storage. It caches pages that have a http status code of - 200, use the GET method. Only non-logged in users receive cached - pages. - Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE - variable.''' - - def __init__(self, app, config): - self.app = app - import redis # only import if used - self.redis = redis # we need to reference this within the class - self.redis_exception = redis.exceptions.ConnectionError - self.redis_connection = None - - def __call__(self, environ, start_response): - - def _start_response(status, response_headers, exc_info=None): - # This wrapper allows us to get the status and headers. - environ['CKAN_PAGE_STATUS'] = status - environ['CKAN_PAGE_HEADERS'] = response_headers - return start_response(status, response_headers, exc_info) - - # Only use cache for GET requests - # REMOTE_USER is used by some tests. - if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): - return self.app(environ, start_response) - - # If there is a ckan cookie (or auth_tkt) we avoid the cache. - # We want to allow other cookies like google analytics ones :( - cookie_string = environ.get('HTTP_COOKIE') - if cookie_string: - for cookie in cookie_string.split(';'): - if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): - return self.app(environ, start_response) - - # Make our cache key - key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) - - # Try to connect if we don't have a connection. Doing this here - # allows the redis server to be unavailable at times. - if self.redis_connection is None: - try: - self.redis_connection = self.redis.StrictRedis() - self.redis_connection.flushdb() - except self.redis_exception: - # Connection may have failed at flush so clear it. - self.redis_connection = None - return self.app(environ, start_response) - - # If cached return cached result - try: - result = self.redis_connection.lrange(key, 0, 2) - except self.redis_exception: - # Connection failed so clear it and return the page as normal. - self.redis_connection = None - return self.app(environ, start_response) - - if result: - headers = json.loads(result[1]) - # Convert headers from list to tuples. - headers = [(str(key), str(value)) for key, value in headers] - start_response(str(result[0]), headers) - # Returning a huge string slows down the server. Therefore we - # cut it up into more usable chunks. - page = result[2] - out = [] - total = len(page) - position = 0 - size = 4096 - while position < total: - out.append(page[position:position + size]) - position += size - return out - - # Generate the response from our application. - page = self.app(environ, _start_response) - - # Only cache http status 200 pages - if not environ['CKAN_PAGE_STATUS'].startswith('200'): - return page - - cachable = False - if environ.get('CKAN_PAGE_CACHABLE'): - cachable = True - - # Cache things if cachable. - if cachable: - # Make sure we consume any file handles etc. - page_string = ''.join(list(page)) - # Use a pipe to add page in a transaction. - pipe = self.redis_connection.pipeline() - pipe.rpush(key, environ['CKAN_PAGE_STATUS']) - pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) - pipe.rpush(key, page_string) - pipe.execute() - return page - - -class TrackingMiddleware(object): - - def __init__(self, app, config): - self.app = app - self.engine = sa.create_engine(config.get('sqlalchemy.url')) - - def __call__(self, environ, start_response): - path = environ['PATH_INFO'] - method = environ.get('REQUEST_METHOD') - if path == '/_tracking' and method == 'POST': - # do the tracking - # get the post data - payload = environ['wsgi.input'].read() - parts = payload.split('&') - data = {} - for part in parts: - k, v = part.split('=') - data[k] = urllib2.unquote(v).decode("utf8") - start_response('200 OK', [('Content-Type', 'text/html')]) - # we want a unique anonomized key for each user so that we do - # not count multiple clicks from the same user. - key = ''.join([ - environ['HTTP_USER_AGENT'], - environ['REMOTE_ADDR'], - environ.get('HTTP_ACCEPT_LANGUAGE', ''), - environ.get('HTTP_ACCEPT_ENCODING', ''), - ]) - key = hashlib.md5(key).hexdigest() - # store key/data here - sql = '''INSERT INTO tracking_raw - (user_key, url, tracking_type) - VALUES (%s, %s, %s)''' - self.engine.execute(sql, key, data.get('url'), data.get('type')) - return [] - return self.app(environ, start_response) - - -def generate_close_and_callback(iterable, callback, environ): - """ - return a generator that passes through items from iterable - then calls callback(environ). - """ - try: - for item in iterable: - yield item - except GeneratorExit: - if hasattr(iterable, 'close'): - iterable.close() - raise - finally: - callback(environ) - - -def execute_on_completion(application, config, callback): - """ - Call callback(environ) once complete response is sent - """ - def inner(environ, start_response): - try: - result = application(environ, start_response) - except: - callback(environ) - raise - return generate_close_and_callback(result, callback, environ) - return inner - - -def cleanup_pylons_response_string(environ): - try: - msg = 'response cleared by pylons response cleanup middleware' - environ['pylons.controller']._py_object.response._body = msg - except (KeyError, AttributeError): - pass diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py new file mode 100644 index 00000000000..33adb1be313 --- /dev/null +++ b/ckan/config/middleware/__init__.py @@ -0,0 +1,132 @@ +# encoding: utf-8 + +"""WSGI app initialization""" + +import webob + +from werkzeug.test import create_environ, run_wsgi_app +from wsgi_party import WSGIParty + +from ckan.config.middleware.flask_app import make_flask_stack +from ckan.config.middleware.pylons_app import make_pylons_stack + +import logging +log = logging.getLogger(__name__) + +# This monkey-patches the webob request object because of the way it messes +# with the WSGI environ. + +# Start of webob.requests.BaseRequest monkey patch +original_charset__set = webob.request.BaseRequest._charset__set + + +def custom_charset__set(self, charset): + original_charset__set(self, charset) + if self.environ.get('CONTENT_TYPE', '').startswith(';'): + self.environ['CONTENT_TYPE'] = '' + +webob.request.BaseRequest._charset__set = custom_charset__set + +webob.request.BaseRequest.charset = property( + webob.request.BaseRequest._charset__get, + custom_charset__set, + webob.request.BaseRequest._charset__del, + webob.request.BaseRequest._charset__get.__doc__) + +# End of webob.requests.BaseRequest monkey patch + + +def make_app(conf, full_stack=True, static_files=True, **app_conf): + ''' + Initialise both the pylons and flask apps, and wrap them in dispatcher + middleware. + ''' + + pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) + flask_app = make_flask_stack(conf) + + app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, + 'flask_app': flask_app}) + + return app + + +class AskAppDispatcherMiddleware(WSGIParty): + + ''' + Establish a 'partyline' to each provided app. Select which app to call + by asking each if they can handle the requested path at PATH_INFO. + + Used to help transition from Pylons to Flask, and should be removed once + Pylons has been deprecated and all app requests are handled by Flask. + + Each app should handle a call to 'can_handle_request(environ)', responding + with a tuple: + (, , []) + where: + `bool` is True if the app can handle the payload url, + `app` is the wsgi app returning the answer + `origin` is an optional string to determine where in the app the url + will be handled, e.g. 'core' or 'extension'. + + Order of precedence if more than one app can handle a url: + Flask Extension > Pylons Extension > Flask Core > Pylons Core + ''' + + def __init__(self, apps=None, invites=(), ignore_missing_services=False): + # Dict of apps managed by this middleware {: , ...} + self.apps = apps or {} + + # A dict of service name => handler mappings. + self.handlers = {} + + # If True, suppress :class:`NoSuchServiceName` errors. Default: False. + self.ignore_missing_services = ignore_missing_services + + self.send_invitations(apps) + + def send_invitations(self, apps): + '''Call each app at the invite route to establish a partyline. Called + on init.''' + PATH = '/__invite__/' + for app_name, app in apps.items(): + environ = create_environ(path=PATH) + environ[self.partyline_key] = self.operator_class(self) + # A reference to the handling app. Used to id the app when + # responding to a handling request. + environ['partyline_handling_app'] = app_name + run_wsgi_app(app, environ) + + def __call__(self, environ, start_response): + '''Determine which app to call by asking each app if it can handle the + url and method defined on the eviron''' + # :::TODO::: Enforce order of precedence for dispatching to apps here. + + app_name = 'pylons_app' # currently defaulting to pylons app + answers = self.ask_around('can_handle_request', environ) + log.debug('Route support answers for {0} {1}: {2}'.format( + environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), + answers)) + available_handlers = [] + for answer in answers: + if len(answer) == 2: + can_handle, asked_app = answer + origin = 'core' + else: + can_handle, asked_app, origin = answer + if can_handle: + available_handlers.append('{0}_{1}'.format(asked_app, origin)) + + # Enforce order of precedence: + # Flask Extension > Pylons Extension > Flask Core > Pylons Core + if available_handlers: + if 'flask_app_extension' in available_handlers: + app_name = 'flask_app' + elif 'pylons_app_extension' in available_handlers: + app_name = 'pylons_app' + elif 'flask_app_core' in available_handlers: + app_name = 'flask_app' + + log.debug('Serving request via {0} app'.format(app_name)) + environ['ckan.app'] = app_name + return self.apps[app_name](environ, start_response) diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py new file mode 100644 index 00000000000..b141b0f3621 --- /dev/null +++ b/ckan/config/middleware/common_middleware.py @@ -0,0 +1,216 @@ +# encoding: utf-8 + +"""Common middleware used by both Flask and Pylons app stacks.""" + +import urllib2 +import hashlib +import urllib +import json + +import sqlalchemy as sa + +from ckan.lib.i18n import get_locales_from_config + + +class I18nMiddleware(object): + """I18n Middleware selects the language based on the url + eg /fr/home is French""" + def __init__(self, app, config): + self.app = app + self.default_locale = config.get('ckan.locale_default', 'en') + self.local_list = get_locales_from_config() + + def __call__(self, environ, start_response): + # strip the language selector from the requested url + # and set environ variables for the language selected + # CKAN_LANG is the language code eg en, fr + # CKAN_LANG_IS_DEFAULT is set to True or False + # CKAN_CURRENT_URL is set to the current application url + + # We only update once for a request so we can keep + # the language and original url which helps with 404 pages etc + if 'CKAN_LANG' not in environ: + path_parts = environ['PATH_INFO'].split('/') + if len(path_parts) > 1 and path_parts[1] in self.local_list: + environ['CKAN_LANG'] = path_parts[1] + environ['CKAN_LANG_IS_DEFAULT'] = False + # rewrite url + if len(path_parts) > 2: + environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) + else: + environ['PATH_INFO'] = '/' + else: + environ['CKAN_LANG'] = self.default_locale + environ['CKAN_LANG_IS_DEFAULT'] = True + + # Current application url + path_info = environ['PATH_INFO'] + # sort out weird encodings + path_info = \ + '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) + + qs = environ.get('QUERY_STRING') + + if qs: + # sort out weird encodings + qs = urllib.quote(qs, '') + environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) + else: + environ['CKAN_CURRENT_URL'] = path_info + + return self.app(environ, start_response) + + +class RootPathMiddleware(object): + ''' + Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url + config. The routes package uses the SCRIPT_NAME variable and appends to the + path and ckan addes the root url causing a duplication of the root path. + + This is a middleware to ensure that even redirects use this logic. + ''' + def __init__(self, app, config): + self.app = app + + def __call__(self, environ, start_response): + # Prevents the variable interfering with the root_path logic + if 'SCRIPT_NAME' in environ: + environ['SCRIPT_NAME'] = '' + + return self.app(environ, start_response) + + +class PageCacheMiddleware(object): + ''' A simple page cache that can store and serve pages. It uses + Redis as storage. It caches pages that have a http status code of + 200, use the GET method. Only non-logged in users receive cached + pages. + Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE + variable.''' + + def __init__(self, app, config): + self.app = app + import redis # only import if used + self.redis = redis # we need to reference this within the class + self.redis_exception = redis.exceptions.ConnectionError + self.redis_connection = None + + def __call__(self, environ, start_response): + + def _start_response(status, response_headers, exc_info=None): + # This wrapper allows us to get the status and headers. + environ['CKAN_PAGE_STATUS'] = status + environ['CKAN_PAGE_HEADERS'] = response_headers + return start_response(status, response_headers, exc_info) + + # Only use cache for GET requests + # REMOTE_USER is used by some tests. + if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): + return self.app(environ, start_response) + + # If there is a ckan cookie (or auth_tkt) we avoid the cache. + # We want to allow other cookies like google analytics ones :( + cookie_string = environ.get('HTTP_COOKIE') + if cookie_string: + for cookie in cookie_string.split(';'): + if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): + return self.app(environ, start_response) + + # Make our cache key + key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) + + # Try to connect if we don't have a connection. Doing this here + # allows the redis server to be unavailable at times. + if self.redis_connection is None: + try: + self.redis_connection = self.redis.StrictRedis() + self.redis_connection.flushdb() + except self.redis_exception: + # Connection may have failed at flush so clear it. + self.redis_connection = None + return self.app(environ, start_response) + + # If cached return cached result + try: + result = self.redis_connection.lrange(key, 0, 2) + except self.redis_exception: + # Connection failed so clear it and return the page as normal. + self.redis_connection = None + return self.app(environ, start_response) + + if result: + headers = json.loads(result[1]) + # Convert headers from list to tuples. + headers = [(str(key), str(value)) for key, value in headers] + start_response(str(result[0]), headers) + # Returning a huge string slows down the server. Therefore we + # cut it up into more usable chunks. + page = result[2] + out = [] + total = len(page) + position = 0 + size = 4096 + while position < total: + out.append(page[position:position + size]) + position += size + return out + + # Generate the response from our application. + page = self.app(environ, _start_response) + + # Only cache http status 200 pages + if not environ['CKAN_PAGE_STATUS'].startswith('200'): + return page + + cachable = False + if environ.get('CKAN_PAGE_CACHABLE'): + cachable = True + + # Cache things if cachable. + if cachable: + # Make sure we consume any file handles etc. + page_string = ''.join(list(page)) + # Use a pipe to add page in a transaction. + pipe = self.redis_connection.pipeline() + pipe.rpush(key, environ['CKAN_PAGE_STATUS']) + pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) + pipe.rpush(key, page_string) + pipe.execute() + return page + + +class TrackingMiddleware(object): + + def __init__(self, app, config): + self.app = app + self.engine = sa.create_engine(config.get('sqlalchemy.url')) + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'] + method = environ.get('REQUEST_METHOD') + if path == '/_tracking' and method == 'POST': + # do the tracking + # get the post data + payload = environ['wsgi.input'].read() + parts = payload.split('&') + data = {} + for part in parts: + k, v = part.split('=') + data[k] = urllib2.unquote(v).decode("utf8") + start_response('200 OK', [('Content-Type', 'text/html')]) + # we want a unique anonomized key for each user so that we do + # not count multiple clicks from the same user. + key = ''.join([ + environ['HTTP_USER_AGENT'], + environ['REMOTE_ADDR'], + environ.get('HTTP_ACCEPT_LANGUAGE', ''), + environ.get('HTTP_ACCEPT_ENCODING', ''), + ]) + key = hashlib.md5(key).hexdigest() + # store key/data here + sql = '''INSERT INTO tracking_raw + (user_key, url, tracking_type) + VALUES (%s, %s, %s)''' + self.engine.execute(sql, key, data.get('url'), data.get('type')) + return [] + return self.app(environ, start_response) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py new file mode 100644 index 00000000000..8d37e26d1b8 --- /dev/null +++ b/ckan/config/middleware/flask_app.py @@ -0,0 +1,81 @@ +# encoding: utf-8 + +from flask import Flask +from flask import abort as flask_abort +from flask import request as flask_request +from flask import _request_ctx_stack +from werkzeug.exceptions import HTTPException + +from wsgi_party import WSGIParty, HighAndDry + + +import logging +log = logging.getLogger(__name__) + + +def make_flask_stack(conf): + """ This has to pass the flask app through all the same middleware that + Pylons used """ + + app = CKANFlask(__name__) + + @app.route('/hello', methods=['GET']) + def hello_world(): + return 'Hello World, this is served by Flask' + + @app.route('/hello', methods=['POST']) + def hello_world_post(): + return 'Hello World, this was posted to Flask' + + return app + + +class CKANFlask(Flask): + + '''Extend the Flask class with a special view to join the 'partyline' + established by AskAppDispatcherMiddleware. + + Also provide a 'can_handle_request' method. + ''' + + def __init__(self, import_name, *args, **kwargs): + super(CKANFlask, self).__init__(import_name, *args, **kwargs) + self.add_url_rule('/__invite__/', endpoint='partyline', + view_func=self.join_party) + self.partyline = None + self.partyline_connected = False + self.invitation_context = None + self.app_name = None # A label for the app handling this request + # (this app). + + def join_party(self, request=flask_request): + # Bootstrap, turn the view function into a 404 after registering. + if self.partyline_connected: + # This route does not exist at the HTTP level. + flask_abort(404) + self.invitation_context = _request_ctx_stack.top + self.partyline = request.environ.get(WSGIParty.partyline_key) + self.app_name = request.environ.get('partyline_handling_app') + self.partyline.connect('can_handle_request', self.can_handle_request) + self.partyline_connected = True + return 'ok' + + def can_handle_request(self, environ): + ''' + Decides whether it can handle a request with the Flask app by + matching the request environ against the route mapper + + Returns (True, 'flask_app') if this is the case. + ''' + + # TODO: identify matching urls as core or extension. This will depend + # on how we setup routing in Flask + + urls = self.url_map.bind_to_environ(environ) + try: + endpoint, args = urls.match() + log.debug('Flask route match, endpoint: {0}, args: {1}'.format( + endpoint, args)) + return (True, self.app_name) + except HTTPException: + raise HighAndDry() diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py new file mode 100644 index 00000000000..4d2af590c0c --- /dev/null +++ b/ckan/config/middleware/pylons_app.py @@ -0,0 +1,218 @@ +# encoding: utf-8 + +import os + +from pylons import config +from pylons.wsgiapp import PylonsApp + +from beaker.middleware import CacheMiddleware, SessionMiddleware +from paste.cascade import Cascade +from paste.registry import RegistryManager +from paste.urlparser import StaticURLParser +from paste.deploy.converters import asbool +from pylons.middleware import ErrorHandler, StatusCodeRedirect +from routes.middleware import RoutesMiddleware +from repoze.who.config import WhoConfig +from repoze.who.middleware import PluggableAuthenticationMiddleware +from fanstatic import Fanstatic + +from ckan.plugins import PluginImplementations +from ckan.plugins.interfaces import IMiddleware +import ckan.lib.uploader as uploader +from ckan.config.environment import load_environment +import ckan.lib.app_globals as app_globals +from ckan.config.middleware import common_middleware + +import logging +log = logging.getLogger(__name__) + + +def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): + """Create a Pylons WSGI application and return it + + ``conf`` + The inherited configuration for this application. Normally from + the [DEFAULT] section of the Paste ini file. + + ``full_stack`` + Whether this application provides a full WSGI stack (by default, + meaning it handles its own exceptions and errors). Disable + full_stack when this application is "managed" by another WSGI + middleware. + + ``static_files`` + Whether this application serves its own static files; disable + when another web server is responsible for serving them. + + ``app_conf`` + The application's local configuration. Normally specified in + the [app:] section of the Paste ini file (where + defaults to main). + + """ + # Configure the Pylons environment + load_environment(conf, app_conf) + + # The Pylons WSGI app + app = PylonsApp() + # set pylons globals + app_globals.reset() + + for plugin in PluginImplementations(IMiddleware): + app = plugin.make_middleware(app, config) + + # Routing/Session/Cache Middleware + app = RoutesMiddleware(app, config['routes.map']) + # we want to be able to retrieve the routes middleware to be able to update + # the mapper. We store it in the pylons config to allow this. + config['routes.middleware'] = app + app = SessionMiddleware(app, config) + app = CacheMiddleware(app, config) + + # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) + # app = QueueLogMiddleware(app) + if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): + app = execute_on_completion(app, config, + cleanup_pylons_response_string) + + # Fanstatic + if asbool(config.get('debug', False)): + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': True, + 'minified': False, + 'bottom': True, + 'bundle': False, + } + else: + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': False, + 'minified': True, + 'bottom': True, + 'bundle': True, + } + app = Fanstatic(app, **fanstatic_config) + + for plugin in PluginImplementations(IMiddleware): + try: + app = plugin.make_error_log_middleware(app, config) + except AttributeError: + log.critical('Middleware class {0} is missing the method' + 'make_error_log_middleware.' + .format(plugin.__class__.__name__)) + + if asbool(full_stack): + # Handle Python exceptions + app = ErrorHandler(app, conf, **config['pylons.errorware']) + + # Display error documents for 400, 403, 404 status codes (and + # 500 when debug is disabled) + if asbool(config['debug']): + app = StatusCodeRedirect(app, [400, 403, 404]) + else: + app = StatusCodeRedirect(app, [400, 403, 404, 500]) + + # Initialize repoze.who + who_parser = WhoConfig(conf['here']) + who_parser.parse(open(app_conf['who.config_file'])) + + app = PluggableAuthenticationMiddleware( + app, + who_parser.identifiers, + who_parser.authenticators, + who_parser.challengers, + who_parser.mdproviders, + who_parser.request_classifier, + who_parser.challenge_decider, + logging.getLogger('repoze.who'), + logging.WARN, # ignored + who_parser.remote_user_key + ) + + # Establish the Registry for this application + app = RegistryManager(app) + + app = common_middleware.I18nMiddleware(app, config) + + if asbool(static_files): + # Serve static files + static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ + else int(config.get('ckan.static_max_age', 3600)) + + static_app = StaticURLParser(config['pylons.paths']['static_files'], + cache_max_age=static_max_age) + static_parsers = [static_app, app] + + storage_directory = uploader.get_storage_path() + if storage_directory: + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + # errno 17 is file already exists + if e.errno != 17: + raise + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + + # Configurable extra static file paths + extra_static_parsers = [] + for public_path in config.get('extra_public_paths', '').split(','): + if public_path.strip(): + extra_static_parsers.append( + StaticURLParser(public_path.strip(), + cache_max_age=static_max_age) + ) + app = Cascade(extra_static_parsers + static_parsers) + + # Page cache + if asbool(config.get('ckan.page_cache_enabled')): + app = common_middleware.PageCacheMiddleware(app, config) + + # Tracking + if asbool(config.get('ckan.tracking_enabled', 'false')): + app = common_middleware.TrackingMiddleware(app, config) + + app = common_middleware.RootPathMiddleware(app, config) + + return app + + +def generate_close_and_callback(iterable, callback, environ): + """ + return a generator that passes through items from iterable + then calls callback(environ). + """ + try: + for item in iterable: + yield item + except GeneratorExit: + if hasattr(iterable, 'close'): + iterable.close() + raise + finally: + callback(environ) + + +def execute_on_completion(application, config, callback): + """ + Call callback(environ) once complete response is sent + """ + def inner(environ, start_response): + try: + result = application(environ, start_response) + except: + callback(environ) + raise + return generate_close_and_callback(result, callback, environ) + return inner + + +def cleanup_pylons_response_string(environ): + try: + msg = 'response cleared by pylons response cleanup middleware' + environ['pylons.controller']._py_object.response._body = msg + except (KeyError, AttributeError): + pass diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 623341185b7..45478e322b9 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -8,7 +8,8 @@ import ckan.plugins as p import ckan.tests.helpers as helpers -from ckan.config.middleware import AskAppDispatcherMiddleware, CKANFlask +from ckan.config.middleware import AskAppDispatcherMiddleware +from ckan.config.middleware.flask_app import CKANFlask from ckan.controllers.partyline import PartylineController From 82fe4fd9bbe50c7e8285968f2caeb4688fc19f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 17:43:16 +0200 Subject: [PATCH 03/18] [#3126] remove backported `subprocess.check_output` --- ckan/lib/util.py | 47 ----------------------------- ckan/tests/test_coding_standards.py | 10 +++--- doc/conf.py | 4 +-- 3 files changed, 7 insertions(+), 54 deletions(-) delete mode 100644 ckan/lib/util.py diff --git a/ckan/lib/util.py b/ckan/lib/util.py deleted file mode 100644 index 3cf19c348d1..00000000000 --- a/ckan/lib/util.py +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 - -'''Shared utility functions for any Python code to use. - -Unlike :py:mod:`ckan.lib.helpers`, the functions in this module are not -available to templates. - -''' -import subprocess - - -# We implement our own check_output() function because -# subprocess.check_output() isn't in Python 2.6. -# This code is copy-pasted from Python 2.7 and adapted to make it work with -# Python 2.6. -# http://hg.python.org/cpython/file/d37f963394aa/Lib/subprocess.py#l544 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 8726f277bd3..6010449676d 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -15,8 +15,6 @@ import re import subprocess -import ckan.lib.util as util - def test_building_the_docs(): '''There should be no warnings or errors when building the Sphinx docs. @@ -28,8 +26,12 @@ def test_building_the_docs(): ''' try: - output = util.check_output( - ['python', 'setup.py', 'build_sphinx', '--all-files', '--fresh-env'], + output = subprocess.check_output( + ['python', + 'setup.py', + 'build_sphinx', + '--all-files', + '--fresh-env'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: assert False, ( diff --git a/doc/conf.py b/doc/conf.py index 635dba13f16..3ed57700fbb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,8 +18,6 @@ import os import subprocess -import ckan.lib.util as util - # If your extensions (or modules documented by autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -126,7 +124,7 @@ def latest_release_tag(): This requires git to be installed. ''' - git_tags = util.check_output( + git_tags = subprocess.check_output( ['git', 'tag', '-l'], stderr=subprocess.STDOUT).split() # FIXME: We could do more careful pattern matching against ckan-X.Y.Z here. From 1efb47a879877d88250cf1016be2bda48c1ff54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 17:53:28 +0200 Subject: [PATCH 04/18] [#3126] don't check if methodcaller is available - `operator.methodcaller` was implemented in python2.7. Now that ckan only supports 2.7 it is no longer required to check if the import is possible or not. We can besure that it is there. --- ckan/model/extension.py | 12 ++---------- ckan/tests/legacy/test_coding_standards.py | 1 - 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/ckan/model/extension.py b/ckan/model/extension.py index b471534a45a..64c68c8f1fa 100644 --- a/ckan/model/extension.py +++ b/ckan/model/extension.py @@ -4,23 +4,17 @@ Provides bridges between the model and plugin PluginImplementationss """ import logging +from operator import methodcaller from sqlalchemy.orm.interfaces import MapperExtension from sqlalchemy.orm.session import SessionExtension import ckan.plugins as plugins -try: - from operator import methodcaller -except ImportError: - def methodcaller(name, *args, **kwargs): - "Replaces stdlib operator.methodcaller in python <2.6" - def caller(obj): - return getattr(obj, name)(*args, **kwargs) - return caller log = logging.getLogger(__name__) + class ObserverNotifier(object): """ Mixin for hooking into SQLAlchemy @@ -93,7 +87,6 @@ def notify_observers(self, func): for observer in plugins.PluginImplementations(plugins.ISession): func(observer) - def after_begin(self, session, transaction, connection): return self.notify_observers( methodcaller('after_begin', session, transaction, connection) @@ -123,4 +116,3 @@ def after_rollback(self, session): return self.notify_observers( methodcaller('after_rollback', session) ) - diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index b3c556833d5..b9a5f0d3142 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -488,7 +488,6 @@ class TestPep8(object): 'ckan/model/authz.py', 'ckan/model/dashboard.py', 'ckan/model/domain_object.py', - 'ckan/model/extension.py', 'ckan/model/follower.py', 'ckan/model/group.py', 'ckan/model/group_extra.py', From f8bd2cd72abfea9911892543309269d59d2925c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 18:22:45 +0200 Subject: [PATCH 05/18] [#3126] remove assert_(not)_in from helpers - `assert_in` and `assert_not_in` were duplicated in `ckan/tests/legacy/__init__.py` and `ckan.tests.helpers` because they were not available in python2.6 - This commit removes the back porting from those files and just imports from nose.tools directly everywhere --- ckan/tests/controllers/test_api.py | 3 +-- ckan/tests/controllers/test_group.py | 3 +-- ckan/tests/controllers/test_organization.py | 4 ++-- ckan/tests/controllers/test_package.py | 3 +-- ckan/tests/controllers/test_tags.py | 3 +-- ckan/tests/helpers.py | 12 +----------- ckan/tests/legacy/__init__.py | 9 --------- ckan/tests/legacy/functional/api/test_api.py | 4 ++-- ckan/tests/legacy/lib/test_dictization.py | 4 ++-- ckan/tests/legacy/models/test_group.py | 5 ++++- ckan/tests/lib/search/test_index.py | 6 +----- ckan/tests/lib/test_mailer.py | 4 +--- ckanext/example_igroupform/tests/test_controllers.py | 3 +-- ckanext/example_iuploader/test/test_plugin.py | 2 +- ckanext/example_theme/custom_emails/tests.py | 5 +---- 15 files changed, 20 insertions(+), 50 deletions(-) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 9cab9d2c1dd..5050ddb32aa 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -7,10 +7,9 @@ import json from routes import url_for -from nose.tools import assert_equal +from nose.tools import assert_equal, assert_in import ckan.tests.helpers as helpers -from ckan.tests.helpers import assert_in from ckan.tests import factories from ckan import model diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index bae63cafc45..67e06c92ce9 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -1,7 +1,7 @@ # encoding: utf-8 from bs4 import BeautifulSoup -from nose.tools import assert_equal, assert_true +from nose.tools import assert_equal, assert_true, assert_in from routes import url_for @@ -9,7 +9,6 @@ import ckan.model as model from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/controllers/test_organization.py b/ckan/tests/controllers/test_organization.py index b8e038bcf1e..ed8da4a48a8 100644 --- a/ckan/tests/controllers/test_organization.py +++ b/ckan/tests/controllers/test_organization.py @@ -1,12 +1,12 @@ # encoding: utf-8 from bs4 import BeautifulSoup -from nose.tools import assert_equal, assert_true +from nose.tools import assert_equal, assert_true, assert_in from routes import url_for from mock import patch from ckan.tests import factories, helpers -from ckan.tests.helpers import webtest_submit, submit_and_follow, assert_in +from ckan.tests.helpers import webtest_submit, submit_and_follow class TestOrganizationNew(helpers.FunctionalTestBase): diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index aac6ba8b864..853414ed4ef 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -6,6 +6,7 @@ assert_not_equal, assert_raises, assert_true, + assert_in ) from mock import patch, MagicMock @@ -17,10 +18,8 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories -from ckan.tests.helpers import assert_in -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/controllers/test_tags.py b/ckan/tests/controllers/test_tags.py index d8c17933515..77d7e083311 100644 --- a/ckan/tests/controllers/test_tags.py +++ b/ckan/tests/controllers/test_tags.py @@ -3,7 +3,7 @@ import math import string -from nose.tools import assert_equal, assert_true, assert_false +from nose.tools import assert_equal, assert_true, assert_false, assert_in from bs4 import BeautifulSoup from routes import url_for @@ -11,7 +11,6 @@ import ckan.tests.helpers as helpers from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index e43093076f2..7f8a0ff77c5 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -22,6 +22,7 @@ import webtest from pylons import config import nose.tools +from nose.tools import assert_in, assert_not_in import mock import ckan.lib.search as search @@ -30,17 +31,6 @@ import ckan.logic as logic -try: - from nose.tools import assert_in, assert_not_in -except ImportError: - # Python 2.6 doesn't have these, so define them here - def assert_in(a, b, msg=None): - assert a in b, msg or '%r was not in %r' % (a, b) - - def assert_not_in(a, b, msg=None): - assert a not in b, msg or '%r was in %r' % (a, b) - - def reset_db(): '''Reset CKAN's database. diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py index acaafc60b52..fcbf89c7134 100644 --- a/ckan/tests/legacy/__init__.py +++ b/ckan/tests/legacy/__init__.py @@ -355,15 +355,6 @@ def skip_test(*args): def clear_flash(res=None): messages = h._flash.pop_messages() -try: - from nose.tools import assert_in, assert_not_in -except ImportError: - def assert_in(a, b, msg=None): - assert a in b, msg or '%r was not in %r' % (a, b) - def assert_not_in(a, b, msg=None): - assert a not in b, msg or '%r was in %r' % (a, b) - - class StatusCodes: STATUS_200_OK = 200 STATUS_201_CREATED = 201 diff --git a/ckan/tests/legacy/functional/api/test_api.py b/ckan/tests/legacy/functional/api/test_api.py index 7c7a1756008..7f172ed1129 100644 --- a/ckan/tests/legacy/functional/api/test_api.py +++ b/ckan/tests/legacy/functional/api/test_api.py @@ -2,10 +2,10 @@ import json +from nose.tools import assert_in + from ckan.tests.legacy.functional.api.base import * -import ckan.tests.legacy -assert_in = ckan.tests.legacy.assert_in class ApiTestCase(ApiTestCase, ControllerTestCase): diff --git a/ckan/tests/legacy/lib/test_dictization.py b/ckan/tests/legacy/lib/test_dictization.py index 864a22b06c0..6a7cfbf8399 100644 --- a/ckan/tests/legacy/lib/test_dictization.py +++ b/ckan/tests/legacy/lib/test_dictization.py @@ -1,10 +1,10 @@ # encoding: utf-8 -from ckan.tests.legacy import assert_equal, assert_not_in, assert_in +from nose.tools import assert_equal, assert_not_in, assert_in from pprint import pprint, pformat from difflib import unified_diff -import ckan.lib.search as search +import ckan.lib.search as search from ckan.lib.create_test_data import CreateTestData from ckan import model from ckan.lib.dictization import (table_dictize, diff --git a/ckan/tests/legacy/models/test_group.py b/ckan/tests/legacy/models/test_group.py index 9d0e76acd96..a4da7148088 100644 --- a/ckan/tests/legacy/models/test_group.py +++ b/ckan/tests/legacy/models/test_group.py @@ -1,9 +1,12 @@ # encoding: utf-8 -from ckan.tests.legacy import assert_equal, assert_in, assert_not_in, CreateTestData +from nose.tools import assert_in, assert_not_in, assert_equal + +from ckan.tests.legacy import CreateTestData import ckan.model as model + class TestGroup(object): @classmethod diff --git a/ckan/tests/lib/search/test_index.py b/ckan/tests/lib/search/test_index.py index dd783e6d500..dbee706d983 100644 --- a/ckan/tests/lib/search/test_index.py +++ b/ckan/tests/lib/search/test_index.py @@ -3,17 +3,13 @@ import datetime import hashlib import json -import nose.tools import nose +from nose.tools import assert_equal, assert_in, assert_not_in from pylons import config import ckan.lib.search as search import ckan.tests.helpers as helpers -assert_equal = nose.tools.assert_equal -assert_in = helpers.assert_in -assert_not_in = helpers.assert_not_in - class TestSearchIndex(object): diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index b45b41b6e47..143be8373b4 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal, assert_raises +from nose.tools import assert_equal, assert_raises, assert_in from pylons import config from email.mime.text import MIMEText from email.parser import Parser @@ -16,8 +16,6 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories -assert_in = helpers.assert_in - class MailerBase(SmtpServerHarness): diff --git a/ckanext/example_igroupform/tests/test_controllers.py b/ckanext/example_igroupform/tests/test_controllers.py index b6f03eaf7e6..be598a13aae 100644 --- a/ckanext/example_igroupform/tests/test_controllers.py +++ b/ckanext/example_igroupform/tests/test_controllers.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal +from nose.tools import assert_equal, assert_in from routes import url_for import ckan.plugins as plugins @@ -8,7 +8,6 @@ import ckan.model as model from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckanext/example_iuploader/test/test_plugin.py b/ckanext/example_iuploader/test/test_plugin.py index f99dc6b15f9..08daf634d21 100644 --- a/ckanext/example_iuploader/test/test_plugin.py +++ b/ckanext/example_iuploader/test/test_plugin.py @@ -6,6 +6,7 @@ from mock import patch from nose.tools import ( assert_equal, + assert_in, assert_is_instance ) from pyfakefs import fake_filesystem @@ -19,7 +20,6 @@ import ckan.tests.helpers as helpers import ckanext.example_iuploader.plugin as plugin -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckanext/example_theme/custom_emails/tests.py b/ckanext/example_theme/custom_emails/tests.py index 3aaf30e40f4..5c72852d15a 100644 --- a/ckanext/example_theme/custom_emails/tests.py +++ b/ckanext/example_theme/custom_emails/tests.py @@ -11,10 +11,7 @@ from ckan.tests.lib.test_mailer import MailerBase import ckan.tests.helpers as helpers -from nose.tools import assert_equal - - -assert_in = helpers.assert_in +from nose.tools import assert_equal, assert_in class TestExampleCustomEmailsPlugin(MailerBase): From 7ec19e36c7c9049a650ba88f8f4bd310ae4e5fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 18:34:32 +0200 Subject: [PATCH 06/18] [#3126] remove check for skipping tests - removes some code that was used to skip tests and change solr settings under python2.6 which is not tested against anymore --- ckanext/datapusher/tests/test.py | 17 ----------------- ckanext/datastore/tests/test_create.py | 6 ------ ckanext/resourceproxy/tests/test_proxy.py | 9 --------- 3 files changed, 32 deletions(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index 2d44c0b14e1..58802fb07ec 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -4,7 +4,6 @@ import httpretty import httpretty.core import nose -import sys import datetime import pylons @@ -56,19 +55,12 @@ def __getattr__(self, attr): httpretty.core.fakesock.socket = HTTPrettyFix -# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 -if sys.version_info < (2, 7, 0): - import socket - socket.setdefaulttimeout(1) - - class TestDatastoreCreate(tests.WsgiAppCase): sysadmin_user = None normal_user = None @classmethod def setup_class(cls): - wsgiapp = middleware.make_app(config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) if not tests.is_datastore_supported(): @@ -84,20 +76,11 @@ def setup_class(cls): set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) - # Httpretty crashes with Solr on Python 2.6, - # skip the tests - if (sys.version_info[0] == 2 and sys.version_info[1] == 6): - raise nose.SkipTest() - @classmethod def teardown_class(cls): rebuild_all_dbs(cls.Session) p.unload('datastore') p.unload('datapusher') - # Reenable Solr indexing - if (sys.version_info[0] == 2 and sys.version_info[1] == 6 - and not p.plugin_loaded('synchronous_search')): - p.load('synchronous_search') def test_create_ckan_resource_in_package(self): package = model.Package.get('annakarenina') diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index a36340ec82d..8e97355b525 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -22,12 +22,6 @@ from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type -# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 -if sys.version_info < (2, 7, 0): - import socket - socket.setdefaulttimeout(1) - - class TestDatastoreCreateNewTests(object): @classmethod def setup_class(cls): diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 28a382ef713..78413da7685 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -1,6 +1,5 @@ # encoding: utf-8 -import sys import requests import unittest import json @@ -60,20 +59,12 @@ def setup_class(cls): wsgiapp = middleware.make_app(config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) create_test_data.CreateTestData.create() - # Httpretty crashes with Solr on Python 2.6, - # skip the tests - if (sys.version_info[0] == 2 and sys.version_info[1] == 6): - raise nose.SkipTest() @classmethod def teardown_class(cls): config.clear() config.update(cls._original_config) model.repo.rebuild_db() - # Reenable Solr indexing - if (sys.version_info[0] == 2 and sys.version_info[1] == 6 - and not p.plugin_loaded('synchronous_search')): - p.load('synchronous_search') def setUp(self): self.url = 'http://www.ckan.org/static/example.json' From ac69f064ce3a17aee89a1d4c8977fff4c718c700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Mon, 20 Jun 2016 09:55:49 +0200 Subject: [PATCH 07/18] [#3126] only reference py2.7 in documentation --- doc/contributing/python.rst | 2 +- doc/maintaining/installing/install-from-source.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/contributing/python.rst b/doc/contributing/python.rst index b557dae81ff..58509d4f916 100644 --- a/doc/contributing/python.rst +++ b/doc/contributing/python.rst @@ -74,7 +74,7 @@ Imports Logging ------- -We use `the Python standard library's logging module `_ +We use `the Python standard library's logging module `_ to log messages in CKAN, e.g.:: import logging diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 5dc82468219..f5f895333cf 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -33,7 +33,7 @@ wiki page for help): ===================== =============================================== Package Description ===================== =============================================== -Python `The Python programming language, v2.6 or 2.7 `_ +Python `The Python programming language, v2.7 `_ |postgres| `The PostgreSQL database system, v8.4 or newer `_ libpq `The C programmer's interface to PostgreSQL `_ pip `A tool for installing and managing Python packages `_ From 9ee75b530a687e43b9eee4fc049ef412b5ad432a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Mon, 20 Jun 2016 09:58:04 +0200 Subject: [PATCH 08/18] [#3126] remove py2.6 from paster templates --- ckan/pastertemplates/template/+dot+travis.yml_tmpl | 1 - ckan/pastertemplates/template/setup.py_tmpl | 1 - 2 files changed, 2 deletions(-) diff --git a/ckan/pastertemplates/template/+dot+travis.yml_tmpl b/ckan/pastertemplates/template/+dot+travis.yml_tmpl index 536ae829969..cc4b3d7cd1f 100644 --- a/ckan/pastertemplates/template/+dot+travis.yml_tmpl +++ b/ckan/pastertemplates/template/+dot+travis.yml_tmpl @@ -1,7 +1,6 @@ language: python sudo: required python: - - "2.6" - "2.7" env: PGVERSION=9.1 install: diff --git a/ckan/pastertemplates/template/setup.py_tmpl b/ckan/pastertemplates/template/setup.py_tmpl index 844042b0035..d6dafcc4cd5 100644 --- a/ckan/pastertemplates/template/setup.py_tmpl +++ b/ckan/pastertemplates/template/setup.py_tmpl @@ -43,7 +43,6 @@ setup( # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', ], From 1fe29c72b8770a3bc7db097fe2e1f44fd53da984 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Tue, 21 Jun 2016 09:14:38 +0200 Subject: [PATCH 09/18] [#3128] Render resource view descriptions as Markdown. Previously, CKAN rendered resource view descriptions as plain text, in contrast to the help text given in the resource view edit form which says that Markdown is supported. --- ckan/templates/package/snippets/resource_view.html | 2 +- ckan/tests/controllers/test_package.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ckan/templates/package/snippets/resource_view.html b/ckan/templates/package/snippets/resource_view.html index 77cf21a4e82..93f1373bc9d 100644 --- a/ckan/templates/package/snippets/resource_view.html +++ b/ckan/templates/package/snippets/resource_view.html @@ -10,7 +10,7 @@ {{ _("Embed") }} -

{{ resource_view['description'] }}

+

{{ h.render_markdown(resource_view['description']) }}

{% if not to_preview and h.resource_view_is_filterable(resource_view) %} {% snippet 'package/snippets/resource_view_filters.html', resource=resource %} diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index aac6ba8b864..9275be446bc 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -1050,6 +1050,17 @@ def test_inexistent_resource_view_page_returns_not_found_code(self): app = self._get_test_app() app.get(url, status=404) + def test_resource_view_description_is_rendered_as_markdown(self): + resource_view = factories.ResourceView(description="Some **Markdown**") + url = url_for(controller='package', + action='resource_read', + id=resource_view['package_id'], + resource_id=resource_view['resource_id'], + view_id=resource_view['id']) + app = self._get_test_app() + response = app.get(url) + response.mustcontain('Some Markdown') + class TestResourceRead(helpers.FunctionalTestBase): @classmethod From dfd2a9187cde6161fc85ba2aba5feefa2516e227 Mon Sep 17 00:00:00 2001 From: Carl Lange Date: Wed, 22 Jun 2016 00:01:59 +0100 Subject: [PATCH 10/18] Fix Pep8 error (long comment) --- ckanext/datapusher/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/datapusher/cli.py b/ckanext/datapusher/cli.py index 2328b051f14..c5d098c9671 100644 --- a/ckanext/datapusher/cli.py +++ b/ckanext/datapusher/cli.py @@ -63,7 +63,8 @@ def _resubmit_all(self): def _submit_all_packages(self): # submit every package - # for each package in the package list, submit each resource w/ _submit_package + # for each package in the package list, + # submit each resource w/ _submit_package import ckan.model as model package_list = p.toolkit.get_action('package_list') for p_id in package_list({'model': model, 'ignore_auth': True}, {}): From 8396d7c6090defefd5f515054afac9362c70c6ff Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 17 Jun 2016 16:15:33 +0200 Subject: [PATCH 11/18] Add encoding specification. --- bin/running_stats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/running_stats.py b/bin/running_stats.py index 006ce6cc029..a2612f35d0f 100644 --- a/bin/running_stats.py +++ b/bin/running_stats.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + '''Tool for a script to keep track changes performed on a large number of objects. From e7513d1df9bfa0b90347216e7199e3e58b23ed5b Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 17 Jun 2016 16:19:03 +0200 Subject: [PATCH 12/18] Add test to ensure string literals have a `u`, `b` or `ur` prefix. All existing Python source code files are explicitly white-listed. Over time, these should be fixed and delisted. New files should not be added to the list. --- ckan/tests/test_coding_standards.py | 689 +++++++++++++++++++++++++--- 1 file changed, 635 insertions(+), 54 deletions(-) diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 6010449676d..d5129476756 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -1,6 +1,6 @@ # encoding: utf-8 -'''A module for coding standards tests. +u'''A module for coding standards tests. These are tests that are not functional- or unit-testing any particular piece of CKAN code, but are checking coding standards. For example: checking that @@ -9,15 +9,55 @@ ''' +import ast import io import os import os.path import re import subprocess +import sys + + +FILESYSTEM_ENCODING = unicode(sys.getfilesystemencoding() + or sys.getdefaultencoding()) + +HERE = os.path.abspath(os.path.dirname(__file__.decode(FILESYSTEM_ENCODING))) + +PROJECT_ROOT = os.path.normpath(os.path.join(HERE, u'..', u'..')) + +# Directories which are ignored when checking Python source code files +IGNORED_DIRS = [ + u'ckan/include', +] + + +def walk_python_files(): + u''' + Generator that yields all CKAN Python source files. + + Yields 2-tuples containing the filename in absolute and relative (to + the project root) form. + ''' + def _is_dir_ignored(root, d): + if d.startswith(u'.'): + return True + return os.path.join(rel_root, d) in IGNORED_DIRS + + for abs_root, dirnames, filenames in os.walk(PROJECT_ROOT): + rel_root = os.path.relpath(abs_root, PROJECT_ROOT) + if rel_root == u'.': + rel_root = u'' + dirnames[:] = [d for d in dirnames if not _is_dir_ignored(rel_root, d)] + for filename in filenames: + if not filename.endswith(u'.py'): + continue + abs_name = os.path.join(abs_root, filename) + rel_name = os.path.join(rel_root, filename) + yield abs_name, rel_name def test_building_the_docs(): - '''There should be no warnings or errors when building the Sphinx docs. + u'''There should be no warnings or errors when building the Sphinx docs. This test unfortunately does take quite a long time to run - rebuilding the docs from scratch just takes a long time. @@ -27,38 +67,38 @@ def test_building_the_docs(): ''' try: output = subprocess.check_output( - ['python', - 'setup.py', - 'build_sphinx', - '--all-files', - '--fresh-env'], + [b'python', + b'setup.py', + b'build_sphinx', + b'--all-files', + b'--fresh-env'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: assert False, ( - "Building the docs failed with return code: {code}".format( + u"Building the docs failed with return code: {code}".format( code=err.returncode)) - output_lines = output.split('\n') + output_lines = output.split(u'\n') - errors = [line for line in output_lines if 'ERROR' in line] + errors = [line for line in output_lines if u'ERROR' in line] if errors: - assert False, ("Don't add any errors to the Sphinx build: " - "{errors}".format(errors=errors)) + assert False, (u"Don't add any errors to the Sphinx build: " + u"{errors}".format(errors=errors)) - warnings = [line for line in output_lines if 'WARNING' in line] + warnings = [line for line in output_lines if u'WARNING' in line] # Some warnings have been around for a long time and aren't easy to fix. # These are allowed, but no more should be added. allowed_warnings = [ - 'WARNING: duplicate label ckan.auth.create_user_via_web', - 'WARNING: duplicate label ckan.auth.create_unowned_dataset', - 'WARNING: duplicate label ckan.auth.user_create_groups', - 'WARNING: duplicate label ckan.auth.anon_create_dataset', - 'WARNING: duplicate label ckan.auth.user_delete_organizations', - 'WARNING: duplicate label ckan.auth.create_user_via_api', - 'WARNING: duplicate label ckan.auth.create_dataset_if_not_in_organization', - 'WARNING: duplicate label ckan.auth.user_delete_groups', - 'WARNING: duplicate label ckan.auth.user_create_organizations', - 'WARNING: duplicate label ckan.auth.roles_that_cascade_to_sub_groups' + u'WARNING: duplicate label ckan.auth.create_user_via_web', + u'WARNING: duplicate label ckan.auth.create_unowned_dataset', + u'WARNING: duplicate label ckan.auth.user_create_groups', + u'WARNING: duplicate label ckan.auth.anon_create_dataset', + u'WARNING: duplicate label ckan.auth.user_delete_organizations', + u'WARNING: duplicate label ckan.auth.create_user_via_api', + u'WARNING: duplicate label ckan.auth.create_dataset_if_not_in_organization', + u'WARNING: duplicate label ckan.auth.user_delete_groups', + u'WARNING: duplicate label ckan.auth.user_create_organizations', + u'WARNING: duplicate label ckan.auth.roles_that_cascade_to_sub_groups' ] # Remove the allowed warnings from the list of collected warnings. @@ -73,56 +113,597 @@ def test_building_the_docs(): if warning not in warnings_to_remove] if new_warnings: - assert False, ("Don't add any new warnings to the Sphinx build: " - "{warnings}".format(warnings=new_warnings)) + assert False, (u"Don't add any new warnings to the Sphinx build: " + u"{warnings}".format(warnings=new_warnings)) def test_source_files_specify_encoding(): - ''' + u''' Test that *.py files have a PEP 263 UTF-8 encoding specification. Empty files and files that only contain comments are ignored. ''' - root_dir = os.path.join(os.path.dirname(__file__), '..', '..') - test_dirs = ['ckan', 'ckanext'] - ignored_dirs = ['ckan/include'] - pattern = re.compile(r'#.*?coding[:=][ \t]*utf-?8') + pattern = re.compile(ur'#.*?coding[:=][ \t]*utf-?8') decode_errors = [] no_specification = [] - - def check_file(filename): + for abs_path, rel_path in walk_python_files(): try: - with io.open(filename, encoding='utf-8') as f: + with io.open(abs_path, encoding=u'utf-8') as f: for line in f: line = line.strip() if pattern.match(line): # Pattern found - return - elif line and not line.startswith('#'): + break + elif line and not line.startswith(u'#'): # File contains non-empty non-comment line - no_specification.append(os.path.relpath(filename, - root_dir)) - return + no_specification.append(rel_path) + break except UnicodeDecodeError: - decode_errors.append(filename) - - for test_dir in test_dirs: - base_dir = os.path.join(root_dir, test_dir) - for root, dirnames, filenames in os.walk(base_dir): - dirnames[:] = [d for d in dirnames if not - os.path.relpath(os.path.join(root, d), root_dir) - in ignored_dirs] - for filename in filenames: - if not filename.endswith('.py'): - continue - check_file(os.path.join(root, filename)) + decode_errors.append(rel_path) msgs = [] if no_specification: - msgs.append('The following files are missing an encoding ' - + 'specification: {}'.format(no_specification)) + msgs.append(u'The following files are missing an encoding ' + + u'specification: {}'.format(no_specification)) if decode_errors: - msgs.append('The following files are not valid UTF-8: {}'.format( + msgs.append(u'The following files are not valid UTF-8: {}'.format( decode_errors)) if msgs: - assert False, '\n\n'.join(msgs) + assert False, u'\n\n'.join(msgs) + + +def renumerate(it): + u''' + Reverse enumerate. + + Yields tuples ``(i, x)`` where ``x`` are the items of ``it`` in + reverse order and ``i`` is the corresponding (decreasing) index. + ``it`` must support ``len``. + ''' + return zip(xrange(len(it) - 1, -1, -1), reversed(it)) + + +def find_unprefixed_string_literals(filename): + u''' + Find unprefixed string literals in a Python source file. + + Returns a list of ``(line_number, column)`` tuples (both 1-based) of + positions where string literals without a ``u`` or ``b`` prefix + start. + + Note: Due to limitations in Python's ``ast`` module this does not + check the rear parts of auto-concatenated string literals + (``'foo' 'bar'``). + ''' + with io.open(filename, encoding=u'utf-8') as f: + lines = f.readlines() + # In some versions of Python, the ast module cannot deal with + # encoding declarations (http://bugs.python.org/issue22221). We + # therefore replace all comment lines at the beginning of the file + # with empty lines (to keep the line numbers correct). + for i, line in enumerate(lines): + line = line.strip() + if line.startswith(u'#'): + lines[i] = u'\n' + elif line: + break + root = ast.parse(u''.join(lines), filename.encode(FILESYSTEM_ENCODING)) + problems = [] + for node in ast.walk(root): + if isinstance(node, ast.Str): + lineno = node.lineno - 1 + col_offset = node.col_offset + if col_offset == -1: + # `lineno` and `col_offset` are broken for literals that span + # multiple lines: For these, `lineno` contains the line of the + # *closing* quotes, and `col_offset` is always -1, see + # https://bugs.python.org/issue16806. We therefore have to + # find the start of the literal manually, which is difficult + # since '''-literals can contain """ and vice versa. The + # following code assumes that no ''' or """ literal begins on + # the same line where a multi-line literal ends. + last_line = lines[lineno] + if last_line.rfind(u'"""') > last_line.rfind(u"'''"): + quotes = u'"""' + else: + quotes = u"'''" + for lineno, line in renumerate(lines[:lineno]): + try: + i = line.rindex(quotes) + if (i > 1) and (line[i - 2:i].lower() == u'ur'): + col_offset = i - 2 + elif (i > 0) and (line[i - 1].lower() in u'rbu'): + col_offset = i - 1 + else: + col_offset = 0 + break + except ValueError: + continue + first_char = lines[lineno][col_offset] + if first_char not in u'ub': # Don't allow capital U and B either + problems.append((lineno + 1, col_offset + 1)) + return sorted(problems) + + +# List of files white-listed for the string literal prefix test. Files on the +# list are expected to be fixed over time and removed from the list. DO NOT ADD +# NEW FILES TO THE LIST. +_STRING_LITERALS_WHITELIST = [ + u'bin/running_stats.py', + u'ckan/__init__.py', + u'ckan/authz.py', + u'ckan/ckan_nose_plugin.py', + u'ckan/config/environment.py', + u'ckan/config/install.py', + u'ckan/config/middleware/__init__.py', + u'ckan/config/middleware/common_middleware.py', + u'ckan/config/middleware/flask_app.py', + u'ckan/config/middleware/pylons_app.py', + u'ckan/config/routing.py', + u'ckan/controllers/admin.py', + u'ckan/controllers/api.py', + u'ckan/controllers/error.py', + u'ckan/controllers/feed.py', + u'ckan/controllers/group.py', + u'ckan/controllers/home.py', + u'ckan/controllers/organization.py', + u'ckan/controllers/package.py', + u'ckan/controllers/partyline.py', + u'ckan/controllers/revision.py', + u'ckan/controllers/storage.py', + u'ckan/controllers/tag.py', + u'ckan/controllers/template.py', + u'ckan/controllers/user.py', + u'ckan/controllers/util.py', + u'ckan/exceptions.py', + u'ckan/i18n/check_po_files.py', + u'ckan/lib/activity_streams.py', + u'ckan/lib/activity_streams_session_extension.py', + u'ckan/lib/alphabet_paginate.py', + u'ckan/lib/app_globals.py', + u'ckan/lib/auth_tkt.py', + u'ckan/lib/authenticator.py', + u'ckan/lib/base.py', + u'ckan/lib/captcha.py', + u'ckan/lib/celery_app.py', + u'ckan/lib/cli.py', + u'ckan/lib/config_tool.py', + u'ckan/lib/create_test_data.py', + u'ckan/lib/datapreview.py', + u'ckan/lib/dictization/__init__.py', + u'ckan/lib/dictization/model_dictize.py', + u'ckan/lib/dictization/model_save.py', + u'ckan/lib/email_notifications.py', + u'ckan/lib/extract.py', + u'ckan/lib/fanstatic_extensions.py', + u'ckan/lib/fanstatic_resources.py', + u'ckan/lib/formatters.py', + u'ckan/lib/hash.py', + u'ckan/lib/helpers.py', + u'ckan/lib/i18n.py', + u'ckan/lib/jinja_extensions.py', + u'ckan/lib/jsonp.py', + u'ckan/lib/mailer.py', + u'ckan/lib/maintain.py', + u'ckan/lib/munge.py', + u'ckan/lib/navl/__init__.py', + u'ckan/lib/navl/dictization_functions.py', + u'ckan/lib/navl/validators.py', + u'ckan/lib/plugins.py', + u'ckan/lib/render.py', + u'ckan/lib/search/__init__.py', + u'ckan/lib/search/common.py', + u'ckan/lib/search/index.py', + u'ckan/lib/search/query.py', + u'ckan/lib/search/sql.py', + u'ckan/lib/uploader.py', + u'ckan/logic/__init__.py', + u'ckan/logic/action/__init__.py', + u'ckan/logic/action/create.py', + u'ckan/logic/action/delete.py', + u'ckan/logic/action/get.py', + u'ckan/logic/action/patch.py', + u'ckan/logic/action/update.py', + u'ckan/logic/auth/__init__.py', + u'ckan/logic/auth/create.py', + u'ckan/logic/auth/delete.py', + u'ckan/logic/auth/get.py', + u'ckan/logic/auth/patch.py', + u'ckan/logic/auth/update.py', + u'ckan/logic/converters.py', + u'ckan/logic/schema.py', + u'ckan/logic/validators.py', + u'ckan/migration/manage.py', + u'ckan/migration/versions/001_add_existing_tables.py', + u'ckan/migration/versions/002_add_author_and_maintainer.py', + u'ckan/migration/versions/003_add_user_object.py', + u'ckan/migration/versions/004_add_group_object.py', + u'ckan/migration/versions/005_add_authorization_tables.py', + u'ckan/migration/versions/006_add_ratings.py', + u'ckan/migration/versions/007_add_system_roles.py', + u'ckan/migration/versions/008_update_vdm_ids.py', + u'ckan/migration/versions/009_add_creation_timestamps.py', + u'ckan/migration/versions/010_add_user_about.py', + u'ckan/migration/versions/011_add_package_search_vector.py', + u'ckan/migration/versions/012_add_resources.py', + u'ckan/migration/versions/013_add_hash.py', + u'ckan/migration/versions/014_hash_2.py', + u'ckan/migration/versions/015_remove_state_object.py', + u'ckan/migration/versions/016_uuids_everywhere.py', + u'ckan/migration/versions/017_add_pkg_relationships.py', + u'ckan/migration/versions/018_adjust_licenses.py', + u'ckan/migration/versions/019_pkg_relationships_state.py', + u'ckan/migration/versions/020_add_changeset.py', + u'ckan/migration/versions/022_add_group_extras.py', + u'ckan/migration/versions/023_add_harvesting.py', + u'ckan/migration/versions/024_add_harvested_document.py', + u'ckan/migration/versions/025_add_authorization_groups.py', + u'ckan/migration/versions/026_authorization_group_user_pk.py', + u'ckan/migration/versions/027_adjust_harvester.py', + u'ckan/migration/versions/028_drop_harvest_source_status.py', + u'ckan/migration/versions/029_version_groups.py', + u'ckan/migration/versions/030_additional_user_attributes.py', + u'ckan/migration/versions/031_move_openid_to_new_field.py', + u'ckan/migration/versions/032_add_extra_info_field_to_resources.py', + u'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', + u'ckan/migration/versions/034_resource_group_table.py', + u'ckan/migration/versions/035_harvesting_doc_versioning.py', + u'ckan/migration/versions/036_lockdown_roles.py', + u'ckan/migration/versions/037_role_anon_editor.py', + u'ckan/migration/versions/038_delete_migration_tables.py', + u'ckan/migration/versions/039_add_expired_id_and_dates.py', + u'ckan/migration/versions/040_reset_key_on_user.py', + u'ckan/migration/versions/041_resource_new_fields.py', + u'ckan/migration/versions/042_user_revision_indexes.py', + u'ckan/migration/versions/043_drop_postgres_search.py', + u'ckan/migration/versions/044_add_task_status.py', + u'ckan/migration/versions/045_user_name_unique.py', + u'ckan/migration/versions/046_drop_changesets.py', + u'ckan/migration/versions/047_rename_package_group_member.py', + u'ckan/migration/versions/048_add_activity_streams_tables.py', + u'ckan/migration/versions/049_add_group_approval_status.py', + u'ckan/migration/versions/050_term_translation_table.py', + u'ckan/migration/versions/051_add_tag_vocabulary.py', + u'ckan/migration/versions/052_update_member_capacities.py', + u'ckan/migration/versions/053_add_group_logo.py', + u'ckan/migration/versions/054_add_resource_created_date.py', + u'ckan/migration/versions/055_update_user_and_activity_detail.py', + u'ckan/migration/versions/056_add_related_table.py', + u'ckan/migration/versions/057_tracking.py', + u'ckan/migration/versions/058_add_follower_tables.py', + u'ckan/migration/versions/059_add_related_count_and_flag.py', + u'ckan/migration/versions/060_add_system_info_table.py', + u'ckan/migration/versions/061_add_follower__group_table.py', + u'ckan/migration/versions/062_add_dashboard_table.py', + u'ckan/migration/versions/063_org_changes.py', + u'ckan/migration/versions/064_add_email_last_sent_column.py', + u'ckan/migration/versions/065_add_email_notifications_preference.py', + u'ckan/migration/versions/066_default_package_type.py', + u'ckan/migration/versions/067_turn_extras_to_strings.py', + u'ckan/migration/versions/068_add_package_extras_index.py', + u'ckan/migration/versions/069_resource_url_and_metadata_modified.py', + u'ckan/migration/versions/070_add_activity_and_resource_indexes.py', + u'ckan/migration/versions/071_add_state_column_to_user_table.py', + u'ckan/migration/versions/072_add_resource_view.py', + u'ckan/migration/versions/073_update_resource_view_resource_id_constraint.py', + u'ckan/migration/versions/074_remove_resource_groups.py', + u'ckan/migration/versions/075_rename_view_plugins.py', + u'ckan/migration/versions/076_rename_view_plugins_2.py', + u'ckan/migration/versions/077_add_revisions_to_system_info.py', + u'ckan/migration/versions/078_remove_old_authz_model.py', + u'ckan/migration/versions/079_resource_revision_index.py', + u'ckan/migration/versions/080_continuity_id_indexes.py', + u'ckan/migration/versions/081_set_datastore_active.py', + u'ckan/migration/versions/082_create_index_creator_user_id.py', + u'ckan/migration/versions/083_remove_related_items.py', + u'ckan/migration/versions/084_add_metadata_created.py', + u'ckan/model/__init__.py', + u'ckan/model/activity.py', + u'ckan/model/core.py', + u'ckan/model/dashboard.py', + u'ckan/model/domain_object.py', + u'ckan/model/extension.py', + u'ckan/model/follower.py', + u'ckan/model/group.py', + u'ckan/model/group_extra.py', + u'ckan/model/license.py', + u'ckan/model/meta.py', + u'ckan/model/misc.py', + u'ckan/model/modification.py', + u'ckan/model/package.py', + u'ckan/model/package_extra.py', + u'ckan/model/package_relationship.py', + u'ckan/model/rating.py', + u'ckan/model/resource.py', + u'ckan/model/resource_view.py', + u'ckan/model/system_info.py', + u'ckan/model/tag.py', + u'ckan/model/task_status.py', + u'ckan/model/term_translation.py', + u'ckan/model/tracking.py', + u'ckan/model/types.py', + u'ckan/model/user.py', + u'ckan/model/vocabulary.py', + u'ckan/pastertemplates/__init__.py', + u'ckan/plugins/core.py', + u'ckan/plugins/interfaces.py', + u'ckan/plugins/toolkit.py', + u'ckan/plugins/toolkit_sphinx_extension.py', + u'ckan/tests/config/test_environment.py', + u'ckan/tests/config/test_middleware.py', + u'ckan/tests/controllers/__init__.py', + u'ckan/tests/controllers/test_admin.py', + u'ckan/tests/controllers/test_api.py', + u'ckan/tests/controllers/test_feed.py', + u'ckan/tests/controllers/test_group.py', + u'ckan/tests/controllers/test_home.py', + u'ckan/tests/controllers/test_organization.py', + u'ckan/tests/controllers/test_package.py', + u'ckan/tests/controllers/test_tags.py', + u'ckan/tests/controllers/test_user.py', + u'ckan/tests/controllers/test_util.py', + u'ckan/tests/factories.py', + u'ckan/tests/helpers.py', + u'ckan/tests/i18n/test_check_po_files.py', + u'ckan/tests/legacy/__init__.py', + u'ckan/tests/legacy/ckantestplugins.py', + u'ckan/tests/legacy/functional/api/__init__.py', + u'ckan/tests/legacy/functional/api/base.py', + u'ckan/tests/legacy/functional/api/model/test_group.py', + u'ckan/tests/legacy/functional/api/model/test_licenses.py', + u'ckan/tests/legacy/functional/api/model/test_package.py', + u'ckan/tests/legacy/functional/api/model/test_ratings.py', + u'ckan/tests/legacy/functional/api/model/test_relationships.py', + u'ckan/tests/legacy/functional/api/model/test_revisions.py', + u'ckan/tests/legacy/functional/api/model/test_tag.py', + u'ckan/tests/legacy/functional/api/model/test_vocabulary.py', + u'ckan/tests/legacy/functional/api/test_activity.py', + u'ckan/tests/legacy/functional/api/test_api.py', + u'ckan/tests/legacy/functional/api/test_dashboard.py', + u'ckan/tests/legacy/functional/api/test_email_notifications.py', + u'ckan/tests/legacy/functional/api/test_follow.py', + u'ckan/tests/legacy/functional/api/test_misc.py', + u'ckan/tests/legacy/functional/api/test_package_search.py', + u'ckan/tests/legacy/functional/api/test_resource.py', + u'ckan/tests/legacy/functional/api/test_resource_search.py', + u'ckan/tests/legacy/functional/api/test_user.py', + u'ckan/tests/legacy/functional/api/test_util.py', + u'ckan/tests/legacy/functional/test_activity.py', + u'ckan/tests/legacy/functional/test_admin.py', + u'ckan/tests/legacy/functional/test_error.py', + u'ckan/tests/legacy/functional/test_group.py', + u'ckan/tests/legacy/functional/test_package.py', + u'ckan/tests/legacy/functional/test_pagination.py', + u'ckan/tests/legacy/functional/test_preview_interface.py', + u'ckan/tests/legacy/functional/test_revision.py', + u'ckan/tests/legacy/functional/test_tag.py', + u'ckan/tests/legacy/functional/test_tracking.py', + u'ckan/tests/legacy/functional/test_user.py', + u'ckan/tests/legacy/html_check.py', + u'ckan/tests/legacy/lib/__init__.py', + u'ckan/tests/legacy/lib/test_alphabet_pagination.py', + u'ckan/tests/legacy/lib/test_authenticator.py', + u'ckan/tests/legacy/lib/test_cli.py', + u'ckan/tests/legacy/lib/test_dictization.py', + u'ckan/tests/legacy/lib/test_dictization_schema.py', + u'ckan/tests/legacy/lib/test_email_notifications.py', + u'ckan/tests/legacy/lib/test_hash.py', + u'ckan/tests/legacy/lib/test_helpers.py', + u'ckan/tests/legacy/lib/test_i18n.py', + u'ckan/tests/legacy/lib/test_navl.py', + u'ckan/tests/legacy/lib/test_resource_search.py', + u'ckan/tests/legacy/lib/test_simple_search.py', + u'ckan/tests/legacy/lib/test_solr_package_search.py', + u'ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py', + u'ckan/tests/legacy/lib/test_solr_schema_version.py', + u'ckan/tests/legacy/lib/test_solr_search_index.py', + u'ckan/tests/legacy/lib/test_tag_search.py', + u'ckan/tests/legacy/logic/test_action.py', + u'ckan/tests/legacy/logic/test_auth.py', + u'ckan/tests/legacy/logic/test_init.py', + u'ckan/tests/legacy/logic/test_member.py', + u'ckan/tests/legacy/logic/test_tag.py', + u'ckan/tests/legacy/logic/test_tag_vocab.py', + u'ckan/tests/legacy/logic/test_validators.py', + u'ckan/tests/legacy/misc/test_format_text.py', + u'ckan/tests/legacy/misc/test_mock_mail_server.py', + u'ckan/tests/legacy/misc/test_sync.py', + u'ckan/tests/legacy/mock_mail_server.py', + u'ckan/tests/legacy/mock_plugin.py', + u'ckan/tests/legacy/models/test_activity.py', + u'ckan/tests/legacy/models/test_extras.py', + u'ckan/tests/legacy/models/test_follower.py', + u'ckan/tests/legacy/models/test_group.py', + u'ckan/tests/legacy/models/test_misc.py', + u'ckan/tests/legacy/models/test_package.py', + u'ckan/tests/legacy/models/test_package_relationships.py', + u'ckan/tests/legacy/models/test_purge_revision.py', + u'ckan/tests/legacy/models/test_resource.py', + u'ckan/tests/legacy/models/test_revision.py', + u'ckan/tests/legacy/models/test_user.py', + u'ckan/tests/legacy/pylons_controller.py', + u'ckan/tests/legacy/schema/test_schema.py', + u'ckan/tests/legacy/test_coding_standards.py', + u'ckan/tests/legacy/test_plugins.py', + u'ckan/tests/legacy/test_versions.py', + u'ckan/tests/lib/__init__.py', + u'ckan/tests/lib/dictization/test_model_dictize.py', + u'ckan/tests/lib/navl/test_dictization_functions.py', + u'ckan/tests/lib/navl/test_validators.py', + u'ckan/tests/lib/search/test_index.py', + u'ckan/tests/lib/test_app_globals.py', + u'ckan/tests/lib/test_auth_tkt.py', + u'ckan/tests/lib/test_base.py', + u'ckan/tests/lib/test_cli.py', + u'ckan/tests/lib/test_config_tool.py', + u'ckan/tests/lib/test_datapreview.py', + u'ckan/tests/lib/test_helpers.py', + u'ckan/tests/lib/test_mailer.py', + u'ckan/tests/lib/test_munge.py', + u'ckan/tests/lib/test_navl.py', + u'ckan/tests/logic/action/__init__.py', + u'ckan/tests/logic/action/test_create.py', + u'ckan/tests/logic/action/test_delete.py', + u'ckan/tests/logic/action/test_get.py', + u'ckan/tests/logic/action/test_patch.py', + u'ckan/tests/logic/action/test_update.py', + u'ckan/tests/logic/auth/__init__.py', + u'ckan/tests/logic/auth/test_create.py', + u'ckan/tests/logic/auth/test_delete.py', + u'ckan/tests/logic/auth/test_get.py', + u'ckan/tests/logic/auth/test_init.py', + u'ckan/tests/logic/auth/test_update.py', + u'ckan/tests/logic/test_conversion.py', + u'ckan/tests/logic/test_converters.py', + u'ckan/tests/logic/test_schema.py', + u'ckan/tests/logic/test_validators.py', + u'ckan/tests/migration/__init__.py', + u'ckan/tests/model/__init__.py', + u'ckan/tests/model/test_license.py', + u'ckan/tests/model/test_resource.py', + u'ckan/tests/model/test_resource_view.py', + u'ckan/tests/model/test_system_info.py', + u'ckan/tests/model/test_user.py', + u'ckan/tests/plugins/__init__.py', + u'ckan/tests/plugins/test_toolkit.py', + u'ckan/tests/test_authz.py', + u'ckan/tests/test_factories.py', + u'ckan/websetup.py', + u'ckanext/datapusher/cli.py', + u'ckanext/datapusher/helpers.py', + u'ckanext/datapusher/interfaces.py', + u'ckanext/datapusher/logic/action.py', + u'ckanext/datapusher/logic/schema.py', + u'ckanext/datapusher/plugin.py', + u'ckanext/datapusher/tests/test.py', + u'ckanext/datapusher/tests/test_action.py', + u'ckanext/datapusher/tests/test_default_views.py', + u'ckanext/datapusher/tests/test_interfaces.py', + u'ckanext/datastore/commands.py', + u'ckanext/datastore/controller.py', + u'ckanext/datastore/db.py', + u'ckanext/datastore/helpers.py', + u'ckanext/datastore/interfaces.py', + u'ckanext/datastore/logic/action.py', + u'ckanext/datastore/logic/auth.py', + u'ckanext/datastore/logic/schema.py', + u'ckanext/datastore/plugin.py', + u'ckanext/datastore/tests/helpers.py', + u'ckanext/datastore/tests/sample_datastore_plugin.py', + u'ckanext/datastore/tests/test_configure.py', + u'ckanext/datastore/tests/test_create.py', + u'ckanext/datastore/tests/test_db.py', + u'ckanext/datastore/tests/test_delete.py', + u'ckanext/datastore/tests/test_disable.py', + u'ckanext/datastore/tests/test_dump.py', + u'ckanext/datastore/tests/test_helpers.py', + u'ckanext/datastore/tests/test_info.py', + u'ckanext/datastore/tests/test_interface.py', + u'ckanext/datastore/tests/test_plugin.py', + u'ckanext/datastore/tests/test_search.py', + u'ckanext/datastore/tests/test_unit.py', + u'ckanext/datastore/tests/test_upsert.py', + u'ckanext/example_iauthfunctions/plugin_v2.py', + u'ckanext/example_iauthfunctions/plugin_v3.py', + u'ckanext/example_iauthfunctions/plugin_v4.py', + u'ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py', + u'ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py', + u'ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py', + u'ckanext/example_iconfigurer/controller.py', + u'ckanext/example_iconfigurer/plugin.py', + u'ckanext/example_iconfigurer/plugin_v1.py', + u'ckanext/example_iconfigurer/plugin_v2.py', + u'ckanext/example_iconfigurer/tests/test_example_iconfigurer.py', + u'ckanext/example_iconfigurer/tests/test_iconfigurer_toolkit.py', + u'ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py', + u'ckanext/example_idatasetform/plugin.py', + u'ckanext/example_idatasetform/plugin_v1.py', + u'ckanext/example_idatasetform/plugin_v2.py', + u'ckanext/example_idatasetform/plugin_v3.py', + u'ckanext/example_idatasetform/plugin_v4.py', + u'ckanext/example_idatasetform/tests/test_controllers.py', + u'ckanext/example_idatasetform/tests/test_example_idatasetform.py', + u'ckanext/example_igroupform/plugin.py', + u'ckanext/example_igroupform/tests/test_controllers.py', + u'ckanext/example_iresourcecontroller/plugin.py', + u'ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py', + u'ckanext/example_itemplatehelpers/plugin.py', + u'ckanext/example_itranslation/plugin.py', + u'ckanext/example_itranslation/plugin_v1.py', + u'ckanext/example_itranslation/tests/test_plugin.py', + u'ckanext/example_iuploader/plugin.py', + u'ckanext/example_iuploader/test/test_plugin.py', + u'ckanext/example_ivalidators/plugin.py', + u'ckanext/example_ivalidators/tests/test_ivalidators.py', + u'ckanext/example_theme/custom_config_setting/plugin.py', + u'ckanext/example_theme/custom_emails/plugin.py', + u'ckanext/example_theme/custom_emails/tests.py', + u'ckanext/example_theme/v01_empty_extension/plugin.py', + u'ckanext/example_theme/v02_empty_template/plugin.py', + u'ckanext/example_theme/v03_jinja/plugin.py', + u'ckanext/example_theme/v04_ckan_extends/plugin.py', + u'ckanext/example_theme/v05_block/plugin.py', + u'ckanext/example_theme/v06_super/plugin.py', + u'ckanext/example_theme/v07_helper_function/plugin.py', + u'ckanext/example_theme/v08_custom_helper_function/plugin.py', + u'ckanext/example_theme/v09_snippet/plugin.py', + u'ckanext/example_theme/v10_custom_snippet/plugin.py', + u'ckanext/example_theme/v11_HTML_and_CSS/plugin.py', + u'ckanext/example_theme/v12_extra_public_dir/plugin.py', + u'ckanext/example_theme/v13_custom_css/plugin.py', + u'ckanext/example_theme/v14_more_custom_css/plugin.py', + u'ckanext/example_theme/v15_fanstatic/plugin.py', + u'ckanext/example_theme/v16_initialize_a_javascript_module/plugin.py', + u'ckanext/example_theme/v17_popover/plugin.py', + u'ckanext/example_theme/v18_snippet_api/plugin.py', + u'ckanext/example_theme/v19_01_error/plugin.py', + u'ckanext/example_theme/v19_02_error_handling/plugin.py', + u'ckanext/example_theme/v20_pubsub/plugin.py', + u'ckanext/example_theme/v21_custom_jquery_plugin/plugin.py', + u'ckanext/imageview/plugin.py', + u'ckanext/imageview/tests/test_view.py', + u'ckanext/multilingual/plugin.py', + u'ckanext/multilingual/tests/test_multilingual_plugin.py', + u'ckanext/reclineview/plugin.py', + u'ckanext/reclineview/tests/test_view.py', + u'ckanext/resourceproxy/controller.py', + u'ckanext/resourceproxy/plugin.py', + u'ckanext/resourceproxy/tests/test_proxy.py', + u'ckanext/stats/__init__.py', + u'ckanext/stats/controller.py', + u'ckanext/stats/plugin.py', + u'ckanext/stats/stats.py', + u'ckanext/stats/tests/__init__.py', + u'ckanext/stats/tests/test_stats_lib.py', + u'ckanext/stats/tests/test_stats_plugin.py', + u'ckanext/test_tag_vocab_plugin.py', + u'ckanext/textview/plugin.py', + u'ckanext/textview/tests/test_view.py', + u'ckanext/webpageview/plugin.py', + u'ckanext/webpageview/tests/test_view.py', + u'doc/conf.py', + u'profile_tests.py', + u'setup.py', +] + + +def test_string_literals_are_prefixed(): + u''' + Test that string literals are prefixed by ``u``, ``b`` or ``ur``. + + See http://docs.ckan.org/en/latest/contributing/unicode.html. + ''' + errors = [] + for abs_path, rel_path in walk_python_files(): + if rel_path in _STRING_LITERALS_WHITELIST: + continue + problems = find_unprefixed_string_literals(abs_path) + if problems: + errors.append((rel_path, problems)) + if errors: + lines = [u'Unprefixed string literals:'] + for filename, problems in errors: + lines.append(u' ' + filename) + for line_no, col_no in problems: + lines.append(u' line {}, column {}'.format(line_no, col_no)) + raise AssertionError(u'\n'.join(lines)) From 2545f74e44aee3e80630f672ce76e74c46d1e101 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 23 Jun 2016 15:38:05 +0100 Subject: [PATCH 13/18] [#3132] Clarify package_relationship_update docs. This action is limited to only updating the comment property. subject, object and type are required to identify the relationship to update. --- ckan/logic/action/update.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ae8bf610559..82e2e7f9124 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -385,22 +385,23 @@ def _update_package_relationship(relationship, comment, context): ref_package_by=ref_package_by) return rel_dict + def package_relationship_update(context, data_dict): '''Update a relationship between two datasets (packages). + The subject, object and type parameters are required to identify the + relationship. Only the comment can be updated. + You must be authorized to edit both the subject and the object datasets. - :param id: the id of the package relationship to update - :type id: string :param subject: the name or id of the dataset that is the subject of the - relationship (optional) + relationship :type subject: string :param object: the name or id of the dataset that is the object of the - relationship (optional) + relationship :param type: the type of the relationship, one of ``'depends_on'``, ``'dependency_of'``, ``'derives_from'``, ``'has_derivation'``, ``'links_to'``, ``'linked_from'``, ``'child_of'`` or ``'parent_of'`` - (optional) :type type: string :param comment: a comment about the relationship (optional) :type comment: string @@ -410,7 +411,8 @@ def package_relationship_update(context, data_dict): ''' model = context['model'] - schema = context.get('schema') or schema_.default_update_relationship_schema() + schema = context.get('schema') \ + or schema_.default_update_relationship_schema() id, id2, rel = _get_or_bust(data_dict, ['subject', 'object', 'type']) From bcc47010f0ba4d4a2718ffb3ad510701fc70a060 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 14:07:50 +0100 Subject: [PATCH 14/18] [#2661] Add revision if owner-org updated. package_owner_org_update is called during a package_update to handle updating the owner organization. A package revision is created during this by package_update. If package_owner_org_update is called in isolation, no revision is created and an error occurs in vdm. This commit ensures a revision is created when package_owner_org_update is called outside of a package_update. --- ckan/logic/action/update.py | 13 +++++- ckan/tests/logic/action/test_update.py | 60 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ae8bf610559..3111b397e12 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -292,6 +292,7 @@ def package_update(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) @@ -1020,6 +1021,7 @@ def package_owner_org_update(context, data_dict): :type id: string ''' model = context['model'] + user = context['user'] name_or_id = data_dict.get('id') organization_id = data_dict.get('organization_id') @@ -1039,10 +1041,17 @@ def package_owner_org_update(context, data_dict): org = None pkg.owner_org = None + if context.get('add_revision', True): + rev = model.repo.new_revision() + rev.author = user + if 'message' in context: + rev.message = context['message'] + else: + rev.message = _(u'REST API: Update object %s') % pkg.get("name") members = model.Session.query(model.Member) \ - .filter(model.Member.table_id == pkg.id) \ - .filter(model.Member.capacity == 'organization') + .filter(model.Member.table_id == pkg.id) \ + .filter(model.Member.capacity == 'organization') need_update = True for member_obj in members: diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 73e766368ef..00813019e47 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -858,3 +858,63 @@ def test_user_create_password_hash_not_for_normal_users(self): user_obj = model.User.get(user['id']) assert user_obj.password != 'pretend-this-is-a-valid-hash' + + +class TestPackageOwnerOrgUpdate(object): + + @classmethod + def teardown_class(cls): + helpers.reset_db() + + def setup(self): + helpers.reset_db() + + def test_package_owner_org_added(self): + '''A package without an owner_org can have one added.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset() + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] is None + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org['id'] + + def test_package_owner_org_changed(self): + '''A package with an owner_org can have it changed.''' + + sysadmin = factories.Sysadmin() + org_1 = factories.Organization() + org_2 = factories.Organization() + dataset = factories.Dataset(owner_org=org_1['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org_1['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org_2['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org_2['id'] + + def test_package_owner_org_removed(self): + '''A package with an owner_org can have it removed.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset(owner_org=org['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=None) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org is None From f240a8cf79e4a8d3a2a592f4cd3d6fa934a23ff7 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 15:34:16 +0100 Subject: [PATCH 15/18] [#2661] Add 'add_revision' to ctx in pkg create. package_create also calls package_owner_org_update, and requires the 'add_revision' property adding to the context to prevent package_owner_org_update from creating an unecessary revision. --- ckan/logic/action/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 0a2a175309d..b85ef81f27b 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -197,6 +197,7 @@ def package_create(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) From 0b8f4d4e480f6d002369876e4e0bcc12557ddb92 Mon Sep 17 00:00:00 2001 From: Nick Such Date: Mon, 27 Jun 2016 13:10:32 -0400 Subject: [PATCH 16/18] Fix broken link to Google's Python style guide --- doc/contributing/python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing/python.rst b/doc/contributing/python.rst index 58509d4f916..3e178a6aa2f 100644 --- a/doc/contributing/python.rst +++ b/doc/contributing/python.rst @@ -9,7 +9,7 @@ For Python code style follow `PEP 8`_ plus the guidelines below. Some good links about Python code style: - `Guide to Python `_ from Hitchhiker's -- `Google Python Style Guide `_ +- `Google Python Style Guide `_ .. seealso:: From 9548412999ee3710b3f8d1887625cb99848f9247 Mon Sep 17 00:00:00 2001 From: Michael Fincham Date: Tue, 28 Jun 2016 13:12:59 +1200 Subject: [PATCH 17/18] Change all example uses of domain names to be RFC compliant special-use names --- ckan/config/deployment.ini_tmpl | 6 +++--- doc/maintaining/configuration.rst | 10 +++++----- test.ini | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d25c9773804..50c58ecedbf 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -166,11 +166,11 @@ ckan.hide_activity_from_users = %(ckan.site_id)s ## Email settings -#email_to = you@yourdomain.com -#error_email_from = paste@localhost +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com #smtp.server = localhost #smtp.starttls = False -#smtp.user = your_username@gmail.com +#smtp.user = username@example.com #smtp.password = your_password #smtp.mail_from = diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 57b8503af2a..67e1521f56b 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -1775,7 +1775,7 @@ smtp.server Example:: - smtp.server = smtp.gmail.com:587 + smtp.server = smtp.example.com:587 Default value: ``None`` @@ -1801,7 +1801,7 @@ smtp.user Example:: - smtp.user = your_username@gmail.com + smtp.user = username@example.com Default value: ``None`` @@ -1827,7 +1827,7 @@ smtp.mail_from Example:: - smtp.mail_from = you@yourdomain.com + smtp.mail_from = ckan@example.com Default value: ``None`` @@ -1841,7 +1841,7 @@ email_to Example:: - email_to = you@yourdomain.com + email_to = errors@example.com Default value: ``None`` @@ -1854,7 +1854,7 @@ error_email_from Example:: - error_email_from = paste@localhost + error_email_from = ckan-errors@example.com Default value: ``None`` diff --git a/test.ini b/test.ini index 4578a15b255..2cf9d6722c7 100644 --- a/test.ini +++ b/test.ini @@ -6,9 +6,9 @@ [DEFAULT] debug = true # Uncomment and replace with the address which should receive any error reports -#email_to = you@yourdomain.com +#email_to = errors@example.com smtp_server = localhost -error_email_from = paste@localhost +error_email_from = ckan-errors@example.com [server:main] use = egg:Paste#http From f58e6841d6d6661417a6602b05a6fdc6441908d3 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 29 Jun 2016 12:00:06 +0300 Subject: [PATCH 18/18] Fix autocomplete request url. Wrapped previously hardcoded url into ckan.url function --- .../base/javascript/modules/resource-view-filters-form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js index aa546285b00..2a7cb3f751c 100644 --- a/ckan/public/base/javascript/modules/resource-view-filters-form.js +++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js @@ -10,7 +10,7 @@ ckan.module('resource-view-filters-form', function (jQuery) { width: 'resolve', minimumInputLength: 0, ajax: { - url: '/api/3/action/datastore_search', + url: ckan.url('/api/3/action/datastore_search'), datatype: 'json', quietMillis: 200, cache: true,