[tests]: #955 Lots of test fixes associated with changes to ckan. Tes…

…ts all work now.

ckanext/dgu/forms/formapi.py

@@ -28,19 +28,18 @@ class FormApi(SingletonPlugin):
 
     implements(IRoutes)
 
-    def after_map(self, map):
+    def before_map(self, map):
         for version in ('', '1/'):
             map.connect('/api/%sform/package/create' % version,           controller='ckanext.dgu.forms.formapi:FormController', action='package_create')
             map.connect('/api/%sform/package/edit/:id' % version,         controller='ckanext.dgu.forms.formapi:FormController', action='package_edit')
-            map.connect('/api/%sform/harvestsource/create' % version,     controller='ckanext.dgu.forms.formapi:FormController', action='harvest_source_create')
-            map.connect('/api/%sform/harvestsource/edit/:id' % version,   controller='ckanext.dgu.forms.formapi:FormController', action='harvest_source_edit')
+        map.connect('/api/2/rest/harvestsource/:id',   controller='ckanext.dgu.forms.formapi:FormController', action='harvest_source_view')
         map.connect('/api/2/form/harvestsource/create',   controller='ckanext.dgu.forms.formapi:FormController', action='harvest_source_create')
         map.connect('/api/2/form/harvestsource/edit/:id', controller='ckanext.dgu.forms.formapi:FormController', action='harvest_source_edit')
         map.connect('/api/2/form/package/create',         controller='ckanext.dgu.forms.formapi:Form2Controller', action='package_create')
         map.connect('/api/2/form/package/edit/:id',       controller='ckanext.dgu.forms.formapi:Form2Controller', action='package_edit')
         return map
 
-    def before_map(self, map):
+    def after_map(self, map):
         return map
 
 class ApiError(Exception):
@@ -382,6 +381,33 @@ def _start_ckan_client(cls, api_key, base_location='http://127.0.0.1:5000/api'):
         import ckanclient
         return ckanclient.CkanClient(base_location=base_location, api_key=api_key)
 
+    def _get_harvest_source(self, id):
+        obj = model.HarvestSource.get(id, default=None)
+        return obj
+
+    def harvest_source_view(self, id):
+        obj = self._get_harvest_source(id)
+        if obj is None:
+            response.status_int = 404
+            return ''            
+        response_data = obj.as_dict()
+        last_harvest_status = 'Not yet harvested'
+        overall_status = 'Not yet harvested'
+        jobs = obj.jobs
+        if len(jobs):
+            last_harvest_status = jobs[-1].status+' ('+'%s package(s) added, 0 deleted, %s errors)'%(
+                len(jobs[-1].report['added']),
+                len(jobs[-1].report['errors']),
+            )
+            added = 0
+            errors = 0
+            for job in jobs:
+                added += len(job.report['added'])
+                errors += len(job.report['errors'])
+            overall_status = '%s package(s) added, 0 deleted'%added
+        response_data['status'] =  'Last harvest status: %s. Overall for this source: %s' % (last_harvest_status, overall_status)
+        return self._finish_ok(response_data)
+
     def harvest_source_create(self):
         try:
             # Get the fieldset.

ckanext/dgu/forms/harvest_source.py

@@ -27,8 +27,7 @@ def build_harvest_source_form():
         """
     ))
     builder.set_field_option('url', 'validate', harvest_source_url_validator)
-    builder.set_field_option('url', 'with_html', {'size':'60'})
-
+    builder.set_field_option('url', 'with_html', {'size':'80'})
     builder.set_field_option('description', 'textarea', {'size':'60x5'})
     builder.set_field_text('description', 'Description', literal('''
         You can add your own notes here about what the URL above represents to remind you later.

ckanext/dgu/tests/__init__.py

@@ -1,31 +1,15 @@
 import os
 
-import paste.fixture
-from paste.deploy import appconfig
+from paste.script.appinstall import SetupCommand
 from pylons import config
 
-from ckan import __file__ as ckan_file
-from ckan.config.middleware import make_app
 from ckan.lib.create_test_data import CreateTestData
+from ckan.tests import WsgiAppCase
+
+
+# Invoke websetup with the current config file
+SetupCommand('setup-app').run([config['__file__']])
 
-def apply_fixture_config(config_):
-    local_config = [
-        ('dgu.xmlrpc_username', 'testuser'),
-        ('dgu.xmlrpc_password', 'testpassword'),
-        ('dgu.xmlrpc_domain', 'localhost:8000'), # must match MockDrupal
-        ]
-    config_.update(local_config)    
-
-class WsgiAppCase(object):
-    ckan_config_dir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(ckan_file)), '..'))
-    config_ = appconfig('config:test.ini', relative_to=ckan_config_dir)
-    local_config = [('ckan.plugins', 'dgu_form_api form_api_tester'),]
-    config_.local_conf.update(local_config)
-    config_.local_conf['ckan.plugins'] = 'dgu_form_api form_api_tester'
-    # set test config for dgu_form_api - it is imported before the test modules
-    apply_fixture_config(config_.local_conf)
-    wsgiapp = make_app(config_.global_conf, **config_.local_conf)
-    app = paste.fixture.TestApp(wsgiapp)
 
 class BaseCase(object):
     @staticmethod
@@ -187,11 +171,6 @@ def pkgs(self):
                 ]
         return self._pkgs
 
-test_publishers = {'1': 'National Health Service',
-                   '2': 'Ealing PCT',
-                   '3': 'Department for Education',
-                   '4': 'Department of Energy and Climate Change',
-                   }
 
 class PackageDictUtil(object):
     @classmethod

ckanext/dgu/tests/forms/test_form_api.py

@@ -21,11 +21,10 @@
 # Todo: Test for access control setup. Just checking an object exists in the model doesn't mean it will be presented through the WebUI.
 
 from ckanext.dgu.forms.formapi import FormController
-from ckanext.dgu.tests import WsgiAppCase, test_publishers, MockDrupalCase, apply_fixture_config
-from ckanext.dgu.tests.functional.mock_drupal import get_mock_drupal_config
+from ckanext.dgu.tests import WsgiAppCase, MockDrupalCase
+from ckanext.dgu.testtools import test_publishers
+from ckanext.dgu.testtools.mock_drupal import get_mock_drupal_config
 
-# import of ckan.tests changes config, so need to reset
-apply_fixture_config(config)
 
 class TestDrupalConnection(MockDrupalCase):
     def test_get_url(self):
@@ -236,7 +235,7 @@ def assert_blank_response(self, response):
 
 class FormsApiTestCase(BaseFormsApiCase):
     def setup(self):
-        model.repo.init_db(conditional=True)
+        model.repo.init_db()
         CreateTestData.create()
         self.package_name = u'formsapi'
         self.package_name_alt = u'formsapialt'
@@ -253,7 +252,7 @@ def setup(self):
         self.harvest_source = None
 
     def teardown(self):
-        model.repo.clean_db()
+        model.repo.rebuild_db()
         model.Session.connection().invalidate()
 
     def get_field_names(self, form):
@@ -475,14 +474,14 @@ def test_submit_harvest_source_create_form_invalid(self):
         assert not self.get_harvest_source_by_url(source_url, None)
         res = self.post_harvest_source_create_form(url=source_url, status=[400])
         self.assert_not_header(res, 'Location')
-        assert "Location (required): Please enter a value" in res.body, res.body
+        assert "URL for source of metadata: Please enter a value" in res.body, res.body
         assert not self.get_harvest_source_by_url(source_url, None)
 
         source_url = u' ' # Not '^http://'
         assert not self.get_harvest_source_by_url(source_url, None)
         res = self.post_harvest_source_create_form(url=source_url, status=[400])
         self.assert_not_header(res, 'Location')
-        assert "Location (required): Harvest source URL is invalid" in res.body, res.body
+        assert "URL for source of metadata: Harvest source URL is invalid" in res.body, res.body
         assert not self.get_harvest_source_by_url(source_url, None)
 
     def test_get_harvest_source_edit_form(self):
@@ -520,7 +519,7 @@ def test_submit_harvest_source_edit_form_invalid(self):
         res = self.post_harvest_source_edit_form(self.harvest_source.id, url=alt_source_url, status=[400])
         assert self.get_harvest_source_by_url(source_url, None)
         self.assert_not_header(res, 'Location')
-        assert "Location (required): Please enter a value" in res.body, res.body
+        assert "URL for source of metadata: Please enter a value" in res.body, res.body
 
 
 class TestFormsApi1(Api1TestCase, FormsApiTestCase): pass

ckanext/dgu/tests/forms/test_package_gov3.py

@@ -12,6 +12,7 @@
 
 from ckanext.dgu.forms.package_gov3 import get_gov3_fieldset
 from ckanext.dgu.tests import *
+from ckanext.dgu.testtools import test_publishers
 
 
 def _get_blank_param_dict(pkg=None, fs=None):
@@ -25,7 +26,7 @@ def get_fieldset(**kwargs):
     return get_gov3_fieldset(**kwargs)
 
 
-class TestFieldset(PylonsTestCase, HtmlCheckMethods):
+class TestFieldset(PylonsTestCase, WsgiAppCase, HtmlCheckMethods):
     @classmethod
     def setup_class(self):
         self.fixtures = Gov3Fixtures()
@@ -457,5 +458,5 @@ def test_8_geo_coverage(self):
         model.repo.commit_and_remove()
 
         outpkg = model.Package.by_name(pkg_name)
-        self.assert_equal(outpkg.extras['geographic_coverage'], '111001: Global, Great Britain (England, Scotland, Wales)')
+        assert_equal(outpkg.extras['geographic_coverage'], '111001: Global, Great Britain (England, Scotland, Wales)')
 

ckanext/dgu/tests/functional/test_form_api_tester.py

@@ -8,7 +8,7 @@
 from ckan.lib.create_test_data import CreateTestData
 
 from ckanext.dgu.tests import WsgiAppCase, MockDrupalCase
-from ckanext.dgu.tests.functional.form_api_tester import *
+from ckanext.dgu.testtools.form_api_tester import *
 
 class TestFormApiTester(WsgiAppCase, MockDrupalCase):
     @classmethod

ckanext/dgu/tests/ons/__init__.py

@@ -1,4 +0,0 @@
-from ckan.lib.create_test_data import CreateTestData
-
-def teardown_module():
-    assert not CreateTestData.get_all_data(), 'A test in module %r forgot to clean-up its data: %r' % (__name__, CreateTestData.get_all_data())

ckanext/dgu/tests/ons/test_ons_loader.py

@@ -1,23 +1,25 @@
 import os
 
 from nose.tools import assert_equal
+from nose.plugins.skip import SkipTest
 from pylons import config
 from sqlalchemy.util import OrderedDict
 
 from ckanext.dgu.ons import importer
 from ckanext.dgu.ons.loader import OnsLoader
 from ckanext.tests.test_loader import TestLoaderBase, USER
 from ckan import model
-from ckan.tests import *
+from ckan.tests import CreateTestData, TestSearchIndexer, is_search_supported
 from ckan.tests.wsgi_ckanclient import WsgiCkanClient
 
-
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 SAMPLE_PATH = os.path.join(TEST_DIR, 'samples')
 SAMPLE_FILEPATH_TEMPLATE = os.path.join(SAMPLE_PATH, 'ons_hub_sample%s.xml')
 def sample_filepath(id):
     return SAMPLE_FILEPATH_TEMPLATE % id
 
+if not is_search_supported():
+    raise SkipTest("Search not supported")
 
 class TestOnsLoadBasic(TestLoaderBase):
     @classmethod

ckanext/dgu/testtools/__init__.py

@@ -0,0 +1,5 @@
+test_publishers = {'1': 'National Health Service',
+                   '2': 'Ealing PCT',
+                   '3': 'Department for Education',
+                   '4': 'Department of Energy and Climate Change',
+                   }

ckanext/dgu/tests/functional/form_api_tester.py → ckanext/dgu/testtools/form_api_tester.py

@@ -13,7 +13,7 @@
 class FormApiTester(SingletonPlugin):
     implements(IRoutes, inherit=True)
 
-    test_controller = 'ckanext.dgu.tests.functional.form_api_tester:FormApiTestController'
+    test_controller = 'ckanext.dgu.testtools.form_api_tester:FormApiTestController'
 
     def after_map(self, map):
         ''' Create test i/f over existing api by adding to routes

ckanext/dgu/tests/functional/mock_drupal.py → ckanext/dgu/testtools/mock_drupal.py

@@ -2,8 +2,9 @@
 
 import paste.script
 
+from ckanext.dgu.testtools import test_publishers
+
 def get_mock_drupal_config():
-    from ckanext.dgu.tests import test_publishers
     return {
         'rpc_path': '/services/xmlrpc',
         'rpc_host': 'localhost',
@@ -36,7 +37,6 @@ def run_mock_drupal(self):
         from SimpleXMLRPCServer import SimpleXMLRPCServer
         from SimpleXMLRPCServer import SimpleXMLRPCRequestHandler
         from xmlrpclib import Fault
-        from ckanext.dgu.tests import test_publishers
 
         config = get_mock_drupal_config()
 

setup.cfg

@@ -0,0 +1,2 @@
+[nosetests]
+with-pylons = test.ini

setup.py

@@ -29,7 +29,7 @@
     entry_points="""
         [ckan.plugins]
         dgu_form_api = ckanext.dgu.forms.formapi:FormApi
-        form_api_tester = ckanext.dgu.tests.functional.form_api_tester:FormApiTester
+        form_api_tester = ckanext.dgu.testtools.form_api_tester:FormApiTester
         
         [console_scripts]
         ons_loader = ckanext.dgu.ons:load
@@ -47,7 +47,7 @@
         report=ckanext.dgu.curation:report
 
         [paste.paster_command]
-        mock_drupal = ckanext.dgu.tests.functional.mock_drupal:Command
+        mock_drupal = ckanext.dgu.testtools.mock_drupal:Command
     """,
     test_suite = 'nose.collector',
 )

test.ini

@@ -0,0 +1,61 @@
+#
+# ckan - Pylons testing environment configuration
+#
+# The %(here)s variable will be replaced with the parent directory of this file
+#
+[DEFAULT]
+debug = true
+# Uncomment and replace with the address which should receive any error reports
+#email_to = you@yourdomain.com
+smtp_server = localhost
+error_email_from = paste@localhost
+
+[server:main]
+use = egg:Paste#http
+host = 0.0.0.0
+port = 5000
+
+
+[app:main]
+use = config:../ckan/test-core.ini
+#faster_db_test_hacks = True
+#sqlalchemy.url = sqlite:///
+
+dgu.xmlrpc_username = testuser
+dgu.xmlrpc_password = testpassword
+dgu.xmlrpc_domain = localhost:8000
+ckan.plugins = dgu_form_api form_api_tester
+
+
+# Logging configuration
+[loggers]
+keys = root, ckan, sqlalchemy
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+
+[logger_ckan]
+qualname = ckan
+handlers = 
+level = INFO
+
+[logger_sqlalchemy]
+handlers =
+qualname = sqlalchemy.engine
+level = WARN  
+
+[handler_console]
+class = StreamHandler
+args = (sys.stdout,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s