From b3fd2f0b14241730fbd8882a2f0f652b870de175 Mon Sep 17 00:00:00 2001 From: tobes Date: Sun, 7 Apr 2013 08:57:08 +0100 Subject: [PATCH] Add coding standards tests --- ckan/tests/test_coding_standards.py | 839 ++++++++++++++++++++++++++++ pip-requirements-test.txt | 2 +- 2 files changed, 840 insertions(+), 1 deletion(-) create mode 100644 ckan/tests/test_coding_standards.py diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py new file mode 100644 index 00000000000..c3161f3029a --- /dev/null +++ b/ckan/tests/test_coding_standards.py @@ -0,0 +1,839 @@ +''' +The aim of these tests is to check and improve the coding standards in ckan. +Common issues are tested for here and tests fail if they are discovered in +files that are either new or were previously good. Bad files are +blacklisted to prevent them throwing errors in many cases because of the +number of affected files e.g. PEP8. However if files start to pass a test +will fail and the file should be removed from the blacklist so that it will +then be kept clean in future. + +The idea is to slowly improve the code quality in ckan without having files +deteriourating when they do reach the required standard. + +Please do not add new files to the list as any new files should meet the +current coding standards. Please add comments by files that fail if there +are legitamate reasons for the failure. +''' + +import sys +import os +import re +import cStringIO + +import pep8 + + +# PEP8 File exceptions +# +# The following files have known PEP8 errors. When the files get to a point +# of not having any such errors they should be removed from this list to +# prevent new errors being added to the file. + +PEP8_BLACKLIST_FILES = [ + 'bin/canada.py', + 'bin/ckan-correct-tags.py', + 'bin/ckan-edit-tags.py', + 'bin/ckan-edit-tags2.py', + 'bin/ckan-hmg-breakdown.py', + 'bin/ckan-hmg-update-licenses.py', + 'bin/ckan-rest-edit-tags.py', + 'bin/ckan_edit_local.py', + 'bin/ckan_spam.py', + 'bin/copy-ckan-2-ckan.py', + 'bin/dump-ukgov.py', + 'bin/dump_23_pkgs.py', + 'bin/fixes.py', + 'bin/loadconfig.py', + 'bin/ons-load.py', + 'bin/revision_manager.py', + 'bin/running_stats.py', + 'bin/status.py', + 'bin/talisckan.py', + 'bin/webstore_test.py', + 'ckan/__init__.py', + 'ckan/ckan_nose_plugin.py', + 'ckan/config/environment.py', + 'ckan/config/middleware.py', + 'ckan/config/routing.py', + 'ckan/config/sp_config.py', + 'ckan/controllers/admin.py', + 'ckan/controllers/api.py', + 'ckan/controllers/feed.py', + 'ckan/controllers/group.py', + 'ckan/controllers/home.py', + 'ckan/controllers/organization.py', + 'ckan/controllers/package.py', + 'ckan/controllers/related.py', + 'ckan/controllers/revision.py', + 'ckan/controllers/storage.py', + 'ckan/controllers/tag.py', + 'ckan/controllers/user.py', + 'ckan/exceptions.py', + 'ckan/i18n/check_po_files.py', + 'ckan/include/rcssmin.py', + 'ckan/include/rjsmin.py', + 'ckan/lib/activity_streams.py', + 'ckan/lib/activity_streams_session_extension.py', + 'ckan/lib/alphabet_paginate.py', + 'ckan/lib/app_globals.py', + 'ckan/lib/authenticator.py', + 'ckan/lib/base.py', + 'ckan/lib/captcha.py', + 'ckan/lib/cli.py', + 'ckan/lib/create_test_data.py', + 'ckan/lib/datapreview.py', + 'ckan/lib/dictization/__init__.py', + 'ckan/lib/dictization/model_dictize.py', + 'ckan/lib/dictization/model_save.py', + 'ckan/lib/dumper.py', + 'ckan/lib/email_notifications.py', + 'ckan/lib/extract.py', + 'ckan/lib/fanstatic_extensions.py', + 'ckan/lib/fanstatic_resources.py', + 'ckan/lib/field_types.py', + 'ckan/lib/formatters.py', + 'ckan/lib/hash.py', + 'ckan/lib/help/flash_messages.py', + 'ckan/lib/helpers.py', + 'ckan/lib/i18n.py', + 'ckan/lib/jinja_extensions.py', + 'ckan/lib/jsonp.py', + 'ckan/lib/mailer.py', + 'ckan/lib/maintain.py', + 'ckan/lib/munge.py', + 'ckan/lib/navl/dictization_functions.py', + 'ckan/lib/navl/validators.py', + 'ckan/lib/package_saver.py', + 'ckan/lib/plugins.py', + 'ckan/lib/render.py', + 'ckan/lib/repoze_patch.py', + 'ckan/lib/search/__init__.py', + 'ckan/lib/search/index.py', + 'ckan/lib/search/query.py', + 'ckan/lib/search/sql.py', + 'ckan/logic/__init__.py', + 'ckan/logic/action/__init__.py', + 'ckan/logic/action/create.py', + 'ckan/logic/action/delete.py', + 'ckan/logic/action/get.py', + 'ckan/logic/action/update.py', + 'ckan/logic/auth/__init__.py', + 'ckan/logic/auth/create.py', + 'ckan/logic/auth/delete.py', + 'ckan/logic/auth/get.py', + 'ckan/logic/auth/update.py', + 'ckan/logic/converters.py', + 'ckan/logic/schema.py', + 'ckan/logic/validators.py', + 'ckan/migration/versions/001_add_existing_tables.py', + 'ckan/migration/versions/002_add_author_and_maintainer.py', + 'ckan/migration/versions/003_add_user_object.py', + 'ckan/migration/versions/004_add_group_object.py', + 'ckan/migration/versions/005_add_authorization_tables.py', + 'ckan/migration/versions/006_add_ratings.py', + 'ckan/migration/versions/007_add_system_roles.py', + 'ckan/migration/versions/008_update_vdm_ids.py', + 'ckan/migration/versions/009_add_creation_timestamps.py', + 'ckan/migration/versions/010_add_user_about.py', + 'ckan/migration/versions/011_add_package_search_vector.py', + 'ckan/migration/versions/012_add_resources.py', + 'ckan/migration/versions/013_add_hash.py', + 'ckan/migration/versions/014_hash_2.py', + 'ckan/migration/versions/015_remove_state_object.py', + 'ckan/migration/versions/016_uuids_everywhere.py', + 'ckan/migration/versions/017_add_pkg_relationships.py', + 'ckan/migration/versions/018_adjust_licenses.py', + 'ckan/migration/versions/019_pkg_relationships_state.py', + 'ckan/migration/versions/020_add_changeset.py', + 'ckan/migration/versions/022_add_group_extras.py', + 'ckan/migration/versions/023_add_harvesting.py', + 'ckan/migration/versions/024_add_harvested_document.py', + 'ckan/migration/versions/025_add_authorization_groups.py', + 'ckan/migration/versions/026_authorization_group_user_pk.py', + 'ckan/migration/versions/027_adjust_harvester.py', + 'ckan/migration/versions/028_drop_harvest_source_status.py', + 'ckan/migration/versions/029_version_groups.py', + 'ckan/migration/versions/030_additional_user_attributes.py', + 'ckan/migration/versions/031_move_openid_to_new_field.py', + 'ckan/migration/versions/032_add_extra_info_field_to_resources.py', + 'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', + 'ckan/migration/versions/034_resource_group_table.py', + 'ckan/migration/versions/035_harvesting_doc_versioning.py', + 'ckan/migration/versions/036_lockdown_roles.py', + 'ckan/migration/versions/037_role_anon_editor.py', + 'ckan/migration/versions/038_delete_migration_tables.py', + 'ckan/migration/versions/039_add_expired_id_and_dates.py', + 'ckan/migration/versions/040_reset_key_on_user.py', + 'ckan/migration/versions/041_resource_new_fields.py', + 'ckan/migration/versions/042_user_revision_indexes.py', + 'ckan/migration/versions/043_drop_postgres_search.py', + 'ckan/migration/versions/044_add_task_status.py', + 'ckan/migration/versions/045_user_name_unique.py', + 'ckan/migration/versions/046_drop_changesets.py', + 'ckan/migration/versions/047_rename_package_group_member.py', + 'ckan/migration/versions/048_add_activity_streams_tables.py', + 'ckan/migration/versions/049_add_group_approval_status.py', + 'ckan/migration/versions/050_term_translation_table.py', + 'ckan/migration/versions/051_add_tag_vocabulary.py', + 'ckan/migration/versions/052_update_member_capacities.py', + 'ckan/migration/versions/053_add_group_logo.py', + 'ckan/migration/versions/054_add_resource_created_date.py', + 'ckan/migration/versions/055_update_user_and_activity_detail.py', + 'ckan/migration/versions/056_add_related_table.py', + 'ckan/migration/versions/057_tracking.py', + 'ckan/migration/versions/058_add_follower_tables.py', + 'ckan/migration/versions/059_add_related_count_and_flag.py', + 'ckan/migration/versions/060_add_system_info_table.py', + 'ckan/migration/versions/061_add_follower__group_table.py', + 'ckan/migration/versions/062_add_dashboard_table.py', + 'ckan/migration/versions/063_org_changes.py', + 'ckan/migration/versions/064_add_email_last_sent_column.py', + 'ckan/migration/versions/065_add_email_notifications_preference.py', + 'ckan/migration/versions/066_default_package_type.py', + 'ckan/migration/versions/067_turn_extras_to_strings.py', + 'ckan/misc.py', + 'ckan/model/__init__.py', + 'ckan/model/activity.py', + 'ckan/model/authz.py', + 'ckan/model/dashboard.py', + 'ckan/model/domain_object.py', + 'ckan/model/extension.py', + 'ckan/model/follower.py', + 'ckan/model/group.py', + 'ckan/model/group_extra.py', + 'ckan/model/license.py', + 'ckan/model/meta.py', + 'ckan/model/misc.py', + 'ckan/model/modification.py', + 'ckan/model/package.py', + 'ckan/model/package_extra.py', + 'ckan/model/package_relationship.py', + 'ckan/model/rating.py', + 'ckan/model/related.py', + 'ckan/model/resource.py', + 'ckan/model/system_info.py', + 'ckan/model/tag.py', + 'ckan/model/task_status.py', + 'ckan/model/term_translation.py', + 'ckan/model/test_user.py', + 'ckan/model/tracking.py', + 'ckan/model/types.py', + 'ckan/model/user.py', + 'ckan/model/vocabulary.py', + 'ckan/new_authz.py', + 'ckan/pastertemplates/__init__.py', + 'ckan/plugins/core.py', + 'ckan/plugins/interfaces.py', + 'ckan/plugins/toolkit.py', + 'ckan/poo.py', + 'ckan/rating.py', + 'ckan/templates_legacy/home/__init__.py', + 'ckan/tests/__init__.py', + 'ckan/tests/ckantestplugin/ckantestplugin/__init__.py', + 'ckan/tests/ckantestplugin/setup.py', + 'ckan/tests/ckantestplugins.py', + 'ckan/tests/functional/api/base.py', + 'ckan/tests/functional/api/model/test_group.py', + 'ckan/tests/functional/api/model/test_licenses.py', + 'ckan/tests/functional/api/model/test_package.py', + 'ckan/tests/functional/api/model/test_ratings.py', + 'ckan/tests/functional/api/model/test_relationships.py', + 'ckan/tests/functional/api/model/test_revisions.py', + 'ckan/tests/functional/api/model/test_tag.py', + 'ckan/tests/functional/api/model/test_vocabulary.py', + 'ckan/tests/functional/api/test_activity.py', + 'ckan/tests/functional/api/test_api.py', + 'ckan/tests/functional/api/test_dashboard.py', + 'ckan/tests/functional/api/test_email_notifications.py', + 'ckan/tests/functional/api/test_follow.py', + 'ckan/tests/functional/api/test_misc.py', + 'ckan/tests/functional/api/test_package_search.py', + 'ckan/tests/functional/api/test_resource.py', + 'ckan/tests/functional/api/test_resource_search.py', + 'ckan/tests/functional/api/test_revision_search.py', + 'ckan/tests/functional/api/test_user.py', + 'ckan/tests/functional/api/test_util.py', + 'ckan/tests/functional/base.py', + 'ckan/tests/functional/test_activity.py', + 'ckan/tests/functional/test_admin.py', + 'ckan/tests/functional/test_cors.py', + 'ckan/tests/functional/test_error.py', + 'ckan/tests/functional/test_follow.py', + 'ckan/tests/functional/test_group.py', + 'ckan/tests/functional/test_home.py', + 'ckan/tests/functional/test_package.py', + 'ckan/tests/functional/test_package_relationships.py', + 'ckan/tests/functional/test_pagination.py', + 'ckan/tests/functional/test_preview_interface.py', + 'ckan/tests/functional/test_related.py', + 'ckan/tests/functional/test_revision.py', + 'ckan/tests/functional/test_search.py', + 'ckan/tests/functional/test_storage.py', + 'ckan/tests/functional/test_tag.py', + 'ckan/tests/functional/test_tag_vocab.py', + 'ckan/tests/functional/test_upload.py', + 'ckan/tests/functional/test_user.py', + 'ckan/tests/html_check.py', + 'ckan/tests/lib/__init__.py', + 'ckan/tests/lib/test_accept.py', + 'ckan/tests/lib/test_alphabet_pagination.py', + 'ckan/tests/lib/test_cli.py', + 'ckan/tests/lib/test_datapreview.py', + 'ckan/tests/lib/test_dictization.py', + 'ckan/tests/lib/test_dictization_schema.py', + 'ckan/tests/lib/test_email_notifications.py', + 'ckan/tests/lib/test_field_types.py', + 'ckan/tests/lib/test_hash.py', + 'ckan/tests/lib/test_helpers.py', + 'ckan/tests/lib/test_i18n.py', + 'ckan/tests/lib/test_mailer.py', + 'ckan/tests/lib/test_munge.py', + 'ckan/tests/lib/test_navl.py', + 'ckan/tests/lib/test_resource_search.py', + 'ckan/tests/lib/test_simple_search.py', + 'ckan/tests/lib/test_solr_package_search.py', + 'ckan/tests/lib/test_solr_package_search_synchronous_update.py', + 'ckan/tests/lib/test_solr_schema_version.py', + 'ckan/tests/lib/test_solr_search_index.py', + 'ckan/tests/lib/test_tag_search.py', + 'ckan/tests/logic/test_action.py', + 'ckan/tests/logic/test_auth.py', + 'ckan/tests/logic/test_tag.py', + 'ckan/tests/logic/test_tag_vocab.py', + 'ckan/tests/logic/test_validators.py', + 'ckan/tests/misc/test_format_text.py', + 'ckan/tests/misc/test_mock_mail_server.py', + 'ckan/tests/misc/test_sync.py', + 'ckan/tests/mock_mail_server.py', + 'ckan/tests/mock_plugin.py', + 'ckan/tests/models/test_extras.py', + 'ckan/tests/models/test_group.py', + 'ckan/tests/models/test_license.py', + 'ckan/tests/models/test_misc.py', + 'ckan/tests/models/test_package.py', + 'ckan/tests/models/test_package_relationships.py', + 'ckan/tests/models/test_purge_revision.py', + 'ckan/tests/models/test_resource.py', + 'ckan/tests/models/test_revision.py', + 'ckan/tests/models/test_user.py', + 'ckan/tests/monkey.py', + 'ckan/tests/pylons_controller.py', + 'ckan/tests/schema/test_schema.py', + 'ckan/tests/test_dumper.py', + 'ckan/tests/test_plugins.py', + 'ckan/tests/test_versions.py', + 'ckan/tests/test_wsgi_ckanclient.py', + 'ckan/tests/wsgi_ckanclient.py', + 'ckan/websetup.py', + 'ckanext/datastore/bin/datastore_setup.py', + 'ckanext/datastore/commands.py', + 'ckanext/datastore/db.py', + 'ckanext/datastore/logic/action.py', + 'ckanext/datastore/logic/auth.py', + 'ckanext/datastore/plugin.py', + 'ckanext/datastore/tests/test_configure.py', + 'ckanext/datastore/tests/test_create.py', + 'ckanext/datastore/tests/test_delete.py', + 'ckanext/datastore/tests/test_search.py', + 'ckanext/datastore/tests/test_unit.py', + 'ckanext/datastore/tests/test_upsert.py', + 'ckanext/example_idatasetform/plugin.py', + 'ckanext/example_itemplatehelpers/plugin.py', + 'ckanext/jsonpreview/plugin.py', + 'ckanext/jsonpreview/tests/test_preview.py', + 'ckanext/multilingual/plugin.py', + 'ckanext/multilingual/tests/test_multilingual_plugin.py', + 'ckanext/pdfpreview/plugin.py', + 'ckanext/pdfpreview/tests/test_preview.py', + 'ckanext/reclinepreview/plugin.py', + 'ckanext/reclinepreview/tests/test_preview.py', + 'ckanext/resourceproxy/controller.py', + 'ckanext/resourceproxy/plugin.py', + 'ckanext/resourceproxy/tests/test_proxy.py', + 'ckanext/stats/controller.py', + 'ckanext/stats/plugin.py', + 'ckanext/stats/stats.py', + 'ckanext/stats/tests/__init__.py', + 'ckanext/stats/tests/test_stats_lib.py', + 'ckanext/stats/tests/test_stats_plugin.py', + 'ckanext/test_tag_vocab_plugin.py', + 'ckanext/tests/plugin.py', + 'doc/conf.py', + 'fabfile.py', + 'profile_tests.py', + 'setup.py', +] + +# Import * file exceptions +# +# The following files contain one or more `from ... import *` lines which +# should not be used in ckan where possible. If the files get fixed they +# should be removed from this list. +# +# import * is bad for many reasons and should be avoided. + +IMPORT_STAR_BLACKLIST_FILES = [ + 'bin/ckan_spam.py', + 'ckan/lib/helpers.py', + 'ckan/lib/package_saver.py', + 'ckan/migration/versions/001_add_existing_tables.py', + 'ckan/migration/versions/002_add_author_and_maintainer.py', + 'ckan/migration/versions/003_add_user_object.py', + 'ckan/migration/versions/004_add_group_object.py', + 'ckan/migration/versions/005_add_authorization_tables.py', + 'ckan/migration/versions/006_add_ratings.py', + 'ckan/migration/versions/007_add_system_roles.py', + 'ckan/migration/versions/008_update_vdm_ids.py', + 'ckan/migration/versions/009_add_creation_timestamps.py', + 'ckan/migration/versions/010_add_user_about.py', + 'ckan/migration/versions/011_add_package_search_vector.py', + 'ckan/migration/versions/012_add_resources.py', + 'ckan/migration/versions/013_add_hash.py', + 'ckan/migration/versions/014_hash_2.py', + 'ckan/migration/versions/015_remove_state_object.py', + 'ckan/migration/versions/016_uuids_everywhere.py', + 'ckan/migration/versions/017_add_pkg_relationships.py', + 'ckan/migration/versions/018_adjust_licenses.py', + 'ckan/migration/versions/019_pkg_relationships_state.py', + 'ckan/migration/versions/020_add_changeset.py', + 'ckan/migration/versions/022_add_group_extras.py', + 'ckan/migration/versions/023_add_harvesting.py', + 'ckan/migration/versions/024_add_harvested_document.py', + 'ckan/migration/versions/025_add_authorization_groups.py', + 'ckan/migration/versions/026_authorization_group_user_pk.py', + 'ckan/migration/versions/027_adjust_harvester.py', + 'ckan/migration/versions/028_drop_harvest_source_status.py', + 'ckan/migration/versions/029_version_groups.py', + 'ckan/migration/versions/030_additional_user_attributes.py', + 'ckan/migration/versions/031_move_openid_to_new_field.py', + 'ckan/migration/versions/032_add_extra_info_field_to_resources.py', + 'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', + 'ckan/migration/versions/034_resource_group_table.py', + 'ckan/migration/versions/035_harvesting_doc_versioning.py', + 'ckan/migration/versions/036_lockdown_roles.py', + 'ckan/migration/versions/037_role_anon_editor.py', + 'ckan/migration/versions/038_delete_migration_tables.py', + 'ckan/migration/versions/039_add_expired_id_and_dates.py', + 'ckan/migration/versions/040_reset_key_on_user.py', + 'ckan/migration/versions/041_resource_new_fields.py', + 'ckan/migration/versions/042_user_revision_indexes.py', + 'ckan/migration/versions/043_drop_postgres_search.py', + 'ckan/migration/versions/044_add_task_status.py', + 'ckan/migration/versions/045_user_name_unique.py', + 'ckan/migration/versions/046_drop_changesets.py', + 'ckan/migration/versions/047_rename_package_group_member.py', + 'ckan/migration/versions/048_add_activity_streams_tables.py', + 'ckan/migration/versions/049_add_group_approval_status.py', + 'ckan/migration/versions/050_term_translation_table.py', + 'ckan/migration/versions/051_add_tag_vocabulary.py', + 'ckan/migration/versions/052_update_member_capacities.py', + 'ckan/migration/versions/053_add_group_logo.py', + 'ckan/migration/versions/056_add_related_table.py', + 'ckan/migration/versions/057_tracking.py', + 'ckan/migration/versions/058_add_follower_tables.py', + 'ckan/migration/versions/059_add_related_count_and_flag.py', + 'ckan/migration/versions/060_add_system_info_table.py', + 'ckan/migration/versions/061_add_follower__group_table.py', + 'ckan/migration/versions/062_add_dashboard_table.py', + 'ckan/migration/versions/063_org_changes.py', + 'ckan/migration/versions/064_add_email_last_sent_column.py', + 'ckan/migration/versions/065_add_email_notifications_preference.py', + 'ckan/plugins/__init__.py', + 'ckan/tests/functional/api/base.py', + 'ckan/tests/functional/api/test_api.py', + 'ckan/tests/functional/api/test_misc.py', + 'ckan/tests/functional/api/test_package_search.py', + 'ckan/tests/functional/api/test_resource_search.py', + 'ckan/tests/functional/api/test_revision_search.py', + 'ckan/tests/functional/test_group.py', + 'ckan/tests/functional/test_home.py', + 'ckan/tests/functional/test_package.py', + 'ckan/tests/functional/test_package_relationships.py', + 'ckan/tests/functional/test_tag.py', + 'ckan/tests/lib/test_alphabet_pagination.py', + 'ckan/tests/lib/test_field_types.py', + 'ckan/tests/lib/test_helpers.py', + 'ckan/tests/lib/test_resource_search.py', + 'ckan/tests/lib/test_tag_search.py', + 'ckan/tests/misc/test_sync.py', + 'ckan/tests/models/test_extras.py', + 'ckan/tests/models/test_group.py', + 'ckan/tests/models/test_misc.py', + 'ckan/tests/models/test_package.py', + 'ckan/tests/models/test_package_relationships.py', + 'ckan/tests/models/test_purge_revision.py', + 'ckan/tests/models/test_resource.py', + 'ckan/tests/models/test_revision.py', + 'ckan/tests/models/test_user.py', + 'ckan/tests/pylons_controller.py', + 'ckan/tests/test_dumper.py', + 'ckan/tests/test_wsgi_ckanclient.py', + 'ckanext/stats/stats.py', + 'fabfile.py', +] + +# Ckan import file exceptions +# +# These files contain lines like `from ckan import x` This should not be +# done except from ckan.common which is written specifically to share +# external functions. When files are fixed they should be removed from this +# list. +# +# The reason for this is to try to remove as many of the circular import +# issues that exist. + +CKAN_IMPORTS_BLACKLIST_FILES = [ + 'bin/ckan-hmg-breakdown.py', + 'bin/dump-ukgov.py', + 'ckan/config/middleware.py', + 'ckan/config/routing.py', + 'ckan/controllers/error.py', + 'ckan/controllers/feed.py', + 'ckan/controllers/revision.py', + 'ckan/controllers/storage.py', + 'ckan/lib/authenticator.py', + 'ckan/lib/base.py', + 'ckan/lib/mailer.py', + 'ckan/lib/munge.py', + 'ckan/lib/package_saver.py', + 'ckan/lib/plugins.py', + 'ckan/lib/search/index.py', + 'ckan/lib/search/query.py', + 'ckan/lib/search/sql.py', + 'ckan/logic/__init__.py', + 'ckan/logic/action/__init__.py', + 'ckan/logic/action/create.py', + 'ckan/logic/auth/__init__.py', + 'ckan/logic/auth/delete.py', + 'ckan/logic/auth/get.py', + 'ckan/logic/auth/update.py', + 'ckan/logic/converters.py', + 'ckan/logic/schema.py', + 'ckan/logic/validators.py', + 'ckan/migration/versions/034_resource_group_table.py', + 'ckan/migration/versions/035_harvesting_doc_versioning.py', + 'ckan/model/test_user.py', + 'ckan/plugins/__init__.py', + 'ckan/plugins/core.py', + 'ckan/tests/__init__.py', + 'ckan/tests/ckantestplugin/ckantestplugin/__init__.py', + 'ckan/tests/functional/api/base.py', + 'ckan/tests/functional/api/model/test_group.py', + 'ckan/tests/functional/api/model/test_licenses.py', + 'ckan/tests/functional/api/model/test_package.py', + 'ckan/tests/functional/api/model/test_ratings.py', + 'ckan/tests/functional/api/model/test_relationships.py', + 'ckan/tests/functional/api/model/test_revisions.py', + 'ckan/tests/functional/api/model/test_tag.py', + 'ckan/tests/functional/api/test_api.py', + 'ckan/tests/functional/api/test_follow.py', + 'ckan/tests/functional/api/test_misc.py', + 'ckan/tests/functional/api/test_package_search.py', + 'ckan/tests/functional/api/test_resource.py', + 'ckan/tests/functional/api/test_resource_search.py', + 'ckan/tests/functional/api/test_revision_search.py', + 'ckan/tests/functional/api/test_user.py', + 'ckan/tests/functional/api/test_util.py', + 'ckan/tests/functional/base.py', + 'ckan/tests/functional/test_activity.py', + 'ckan/tests/functional/test_admin.py', + 'ckan/tests/functional/test_cors.py', + 'ckan/tests/functional/test_follow.py', + 'ckan/tests/functional/test_group.py', + 'ckan/tests/functional/test_home.py', + 'ckan/tests/functional/test_package.py', + 'ckan/tests/functional/test_package_relationships.py', + 'ckan/tests/functional/test_pagination.py', + 'ckan/tests/functional/test_revision.py', + 'ckan/tests/functional/test_search.py', + 'ckan/tests/functional/test_storage.py', + 'ckan/tests/functional/test_tag.py', + 'ckan/tests/functional/test_tag_vocab.py', + 'ckan/tests/functional/test_upload.py', + 'ckan/tests/functional/test_user.py', + 'ckan/tests/lib/__init__.py', + 'ckan/tests/lib/test_alphabet_pagination.py', + 'ckan/tests/lib/test_cli.py', + 'ckan/tests/lib/test_dictization.py', + 'ckan/tests/lib/test_dictization_schema.py', + 'ckan/tests/lib/test_field_types.py', + 'ckan/tests/lib/test_hash.py', + 'ckan/tests/lib/test_helpers.py', + 'ckan/tests/lib/test_i18n.py', + 'ckan/tests/lib/test_mailer.py', + 'ckan/tests/lib/test_navl.py', + 'ckan/tests/lib/test_resource_search.py', + 'ckan/tests/lib/test_simple_search.py', + 'ckan/tests/lib/test_solr_package_search.py', + 'ckan/tests/lib/test_solr_package_search_synchronous_update.py', + 'ckan/tests/lib/test_solr_schema_version.py', + 'ckan/tests/lib/test_solr_search_index.py', + 'ckan/tests/lib/test_tag_search.py', + 'ckan/tests/logic/test_action.py', + 'ckan/tests/logic/test_auth.py', + 'ckan/tests/logic/test_tag.py', + 'ckan/tests/logic/test_validators.py', + 'ckan/tests/misc/test_format_text.py', + 'ckan/tests/misc/test_mock_mail_server.py', + 'ckan/tests/misc/test_sync.py', + 'ckan/tests/mock_mail_server.py', + 'ckan/tests/mock_plugin.py', + 'ckan/tests/models/test_extras.py', + 'ckan/tests/models/test_group.py', + 'ckan/tests/models/test_license.py', + 'ckan/tests/models/test_misc.py', + 'ckan/tests/models/test_package.py', + 'ckan/tests/models/test_package_relationships.py', + 'ckan/tests/models/test_purge_revision.py', + 'ckan/tests/models/test_resource.py', + 'ckan/tests/models/test_revision.py', + 'ckan/tests/models/test_user.py', + 'ckan/tests/pylons_controller.py', + 'ckan/tests/schema/test_schema.py', + 'ckan/tests/test_dumper.py', + 'ckan/tests/test_plugins.py', + 'ckan/tests/test_wsgi_ckanclient.py', + 'ckan/websetup.py', + 'ckanext/jsonpreview/tests/test_preview.py', + 'ckanext/multilingual/plugin.py', + 'ckanext/pdfpreview/tests/test_preview.py', + 'ckanext/reclinepreview/tests/test_preview.py', + 'ckanext/stats/controller.py', + 'ckanext/stats/stats.py', + 'ckanext/stats/tests/__init__.py', + 'ckanext/stats/tests/test_stats_lib.py', + 'ckanext/stats/tests/test_stats_plugin.py', + 'ckanext/test_tag_vocab_plugin.py', + 'setup.py', +] + +# Nasty str() issues +# +# There are places in ckan where code is like `'...%s..' % str(..)` these +# cause problems when unicode is present but can remain dormant for a long +# time before the issue is apparent so try to remove these. The value is +# converted to a string anyway so the str() is unneeded in any place. + +NASTY_STR_BLACKLIST_FILES = [ + 'ckan/ckan_nose_plugin.py', + 'ckan/controllers/api.py', + 'ckan/controllers/group.py', + 'ckan/lib/activity_streams.py', + 'ckan/logic/action/create.py', + 'ckan/logic/action/update.py', + 'ckan/logic/auth/create.py', + 'ckan/logic/auth/delete.py', + 'ckan/logic/auth/get.py', + 'ckan/logic/validators.py', + 'ckan/tests/coding_standards.py', # example causes error + 'ckan/tests/functional/api/test_revision_search.py', + 'ckan/tests/functional/test_pagination.py', + 'ckan/tests/models/test_package_relationships.py', + 'ckanext/resourceproxy/controller.py', +] + +file_path = os.path.dirname(__file__) +base_path = os.path.abspath(os.path.join(file_path, '..', '..')) + + +def process_directory(directory): + base_len = len(base_path) + 1 + for (dirpath, dirnames, filenames) in os.walk(directory): + for name in filenames: + if name.endswith('.py'): + path = os.path.join(dirpath, name) + filename = path[base_len:] + yield path, filename + + +def output_errors(filename, errors): + out = [''] + out.append('-' * len(filename)) + out.append(filename) + out.append('-' * len(filename)) + for error in errors: + out.append(error) + return '\n'.join(out) + + +def show_fails(msg, errors): + if errors: + msg = ['\n%s' % msg] + for error in errors: + msg.append(errors[error]) + msg.append('\n\nFailing Files:\n==============') + msg += sorted(errors) + raise Exception('\n'.join(msg)) + + +def show_passing(msg, errors): + if errors: + raise Exception('\n%s\n\n' % msg + '\n'.join(sorted(errors))) + + +class TestNastyString(object): + + fails = {} + passes = [] + done = False + + @classmethod + def setup(cls): + if not cls.done: + cls.process() + cls.done = True + + @classmethod + def process(cls): + blacklist = NASTY_STR_BLACKLIST_FILES + re_import_star = re.compile( + r'''("[^"]*\%s[^"]*"|'[^']*\%s[^']*').*%.*str\(''' + ) + for path, filename in process_directory(base_path): + f = open(path, 'r') + count = 1 + errors = [] + for line in f: + if re_import_star.search(line): + errors.append('ln:%s \t%s' % (count, line[:-1])) + count += 1 + if errors and not filename in blacklist: + cls.fails[filename] = output_errors(filename, errors) + elif not errors and filename in blacklist: + cls.passes.append(filename) + + def test_good(self): + msg = 'The following files passed nasty str() rules' + show_passing(msg, self.passes) + + def test_bad(self): + msg = 'The following files have nasty str() issues that need resolving' + show_fails(msg, self.fails) + + +class TestImportFromCkan(object): + + fails = {} + passes = [] + done = False + + @classmethod + def setup(cls): + if not cls.done: + cls.process() + cls.done = True + + @classmethod + def process(cls): + blacklist = CKAN_IMPORTS_BLACKLIST_FILES + re_import_star = re.compile(r'^from\s.*\bckan\b(?!\.common).*\bimport') + for path, filename in process_directory(base_path): + f = open(path, 'r') + count = 1 + errors = [] + for line in f: + if re_import_star.search(line): + errors.append('ln:%s \t%s' % (count, line[:-1])) + count += 1 + if errors and not filename in blacklist: + cls.fails[filename] = output_errors(filename, errors) + elif not errors and filename in blacklist: + cls.passes.append(filename) + + def test_import_good(self): + msg = 'The following files passed ckan import rules' + show_passing(msg, self.passes) + + def test_import_bad(self): + msg = 'The following files have ckan import issues that need resolving' + show_fails(msg, self.fails) + + +class TestImportStar(object): + + fails = {} + passes = [] + done = False + + @classmethod + def setup(cls): + if not cls.done: + cls.process() + cls.done = True + + @classmethod + def process(cls): + blacklist = IMPORT_STAR_BLACKLIST_FILES + re_import_star = re.compile(r'^from\s+.*\simport\s+\*') + for path, filename in process_directory(base_path): + f = open(path, 'r') + count = 1 + errors = [] + for line in f: + if re_import_star.search(line): + errors.append('%s ln:%s import *\n\t%s' + % (filename, count, line)) + count += 1 + if errors and not filename in blacklist: + cls.fails[filename] = output_errors(filename, errors) + elif not errors and filename in blacklist: + cls.passes.append(filename) + + def test_import_good(self): + msg = 'The following files passed import * rules' + show_passing(msg, self.passes) + + def test_import_bad(self): + msg = 'The following files have import * issues that need resolving' + show_fails(msg, self.fails) + + +class TestPep8(object): + + fails = {} + passes = [] + done = False + + @classmethod + def setup(cls): + if not cls.done: + cls.process() + cls.done = True + + @classmethod + def process(cls): + blacklist = PEP8_BLACKLIST_FILES + for path, filename in process_directory(base_path): + errors = cls.find_pep8_errors(filename=path) + if errors and not filename in blacklist: + cls.fails[filename] = output_errors(filename, errors) + elif not errors and filename in blacklist: + cls.passes.append(filename) + + def test_pep8_fails(self): + msg = 'The following files have pep8 issues that need resolving' + show_fails(msg, self.fails) + + def test_pep8_pass(self): + msg = 'The following files passed pep8 but are blacklisted' + show_passing(msg, self.passes) + + @staticmethod + def find_pep8_errors(filename=None, lines=None): + + try: + sys.stdout = cStringIO.StringIO() + checker = pep8.Checker(filename=filename, lines=lines) + checker.check_all() + output = sys.stdout.getvalue() + finally: + sys.stdout = sys.__stdout__ + + errors = [] + for line in output.split('\n'): + parts = line.split(' ', 2) + if len(parts) == 3: + location, error, desc = parts + line_no = location.split(':')[1] + errors.append('%s ln:%s %s' % (error, line_no, desc)) + return errors diff --git a/pip-requirements-test.txt b/pip-requirements-test.txt index 1bd83ee861e..ed58d11f6a0 100644 --- a/pip-requirements-test.txt +++ b/pip-requirements-test.txt @@ -3,4 +3,4 @@ nose==1.2.1 httpretty==0.5 -e git+https://github.com/okfn/ckanclient#egg=ckanclient - +pep8==1.3.3