Skip to content

Commit

Permalink
Merge branch 'pytest' of https://github.com/DataShades/ckan into Data…
Browse files Browse the repository at this point in the history
…Shades-pytest
  • Loading branch information
amercader committed Dec 16, 2019
2 parents 06af539 + df55858 commit b9df6b6
Show file tree
Hide file tree
Showing 197 changed files with 33,195 additions and 30,964 deletions.
18 changes: 5 additions & 13 deletions .circleci/config.yml
Expand Up @@ -14,7 +14,7 @@ jobs:
CKAN_POSTGRES_PWD: pass
PGPASSWORD: ckan
NODE_TESTS_CONTAINER: 2
NOSETEST_COMMON_OPTIONS: -v --ckan --reset-db --with-pylons=test-core-circle-ci.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=/root/junit/junit.xml ckan ckanext
PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4 --test-group-random-seed 1
- image: postgres:10
environment:
POSTGRES_USER: ckan
Expand Down Expand Up @@ -65,26 +65,18 @@ jobs:
- run: |
mkdir -p ~/junit
case $CIRCLE_NODE_INDEX in
0) nosetests $NOSETEST_COMMON_OPTIONS --segments 0123
0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
;;
1) nosetests $NOSETEST_COMMON_OPTIONS --segments 4567
1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
;;
2) nosetests $NOSETEST_COMMON_OPTIONS --segments 89ab
2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
;;
3) nosetests $NOSETEST_COMMON_OPTIONS --segments cdef
3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
;;
esac
- store_test_results:
path: ~/junit

# test_revision_legacy_code is run separately because it mucks up the model for some other tests when nose starts up
- run:
command: |
case $CIRCLE_NODE_INDEX in
3) nosetests -v --ckan --with-pylons=test-core-circle-ci.ini --nologcapture test_revision_legacy_code.py
;;
esac
# Tests Frontend, only in one container
- run:
command: |
Expand Down
1 change: 0 additions & 1 deletion ckan/cli/dataset.py
Expand Up @@ -53,7 +53,6 @@ def delete(package):
dataset = _get_dataset(package)
old_state = dataset.state

model.repo.new_revision()
dataset.delete()
model.repo.commit_and_remove()
dataset = _get_dataset(package)
Expand Down
2 changes: 1 addition & 1 deletion ckan/logic/action/update.py
Expand Up @@ -747,7 +747,7 @@ def task_status_update(context, data_dict):
'''
model = context['model']
session = model.meta.create_local_session()
session = model.Session
context['session'] = session

user = context['user']
Expand Down
3 changes: 2 additions & 1 deletion ckan/model/__init__.py
Expand Up @@ -187,7 +187,8 @@ def init_db(self):
self.session.remove()
# sqlite database needs to be recreated each time as the
# memory database is lost.
if self.metadata.bind.name == 'sqlite':

if self.metadata.bind.engine.url.drivername == 'sqlite':
# this creates the tables, which isn't required inbetween tests
# that have simply called rebuild_db.
self.create_db()
Expand Down
5 changes: 3 additions & 2 deletions ckan/model/meta.py
Expand Up @@ -100,11 +100,12 @@ def after_rollback(self, session):

def engine_is_sqlite(sa_engine=None):
# Returns true iff the engine is connected to a sqlite database.
return (sa_engine or engine).url.drivername == 'sqlite'

return (sa_engine or engine).engine.url.drivername == 'sqlite'


def engine_is_pg(sa_engine=None):
# Returns true iff the engine is connected to a postgresql database.
# According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql
# all Postgres driver names start with `postgres`
return (sa_engine or engine).url.drivername.startswith('postgres')
return (sa_engine or engine).engine.url.drivername.startswith('postgres')
4 changes: 2 additions & 2 deletions ckan/pastertemplates/template/README.rst_tmpl
Expand Up @@ -100,12 +100,12 @@ Tests

To run the tests, do::

nosetests --nologcapture --with-pylons=test.ini
pytest --ckan-ini=test.ini

To run the tests and produce a coverage report, first make sure you have
coverage installed in your virtualenv (``pip install coverage``) then run::

nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.{{ project_shortname }} --cover-inclusive --cover-erase --cover-tests
pytest --ckan-ini=test.ini --cov=ckanext.{{ project_shortname }}


----------------------------------------
Expand Down
10 changes: 2 additions & 8 deletions ckan/pastertemplates/template/bin/travis-run.sh_tmpl
Expand Up @@ -5,14 +5,8 @@ flake8 --version
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude ckan,{{ project }}

nosetests --ckan \
--nologcapture \
--with-pylons=subdir/test.ini \
--with-coverage \
--cover-package=ckanext.{{ project_shortname }} \
--cover-inclusive \
--cover-erase \
--cover-tests
pytest --ckan-ini=subdir/test.ini \
--cov=ckanext.{{ project_shortname }}

# strict linting
flake8 . --count --max-complexity=10 --max-line-length=127 --statistics --exclude ckan,{{ project }}
189 changes: 89 additions & 100 deletions ckan/tests/config/test_environment.py
@@ -1,113 +1,102 @@
# encoding: utf-8

import os
from nose import tools as nosetools

from ckan.common import config
import pytest

import ckan.tests.helpers as h
import ckan.plugins as p
from ckan.config import environment
from ckan.exceptions import CkanConfigurationException

from ckan.tests import helpers


class TestUpdateConfig(h.FunctionalTestBase):

'''
Tests for config settings from env vars, set in
config.environment.update_config().
'''

ENV_VAR_LIST = [
('CKAN_SQLALCHEMY_URL', 'postgresql://mynewsqlurl/'),
('CKAN_DATASTORE_WRITE_URL', 'http://mynewdbwriteurl/'),
('CKAN_DATASTORE_READ_URL', 'http://mynewdbreadurl/'),
('CKAN_SOLR_URL', 'http://mynewsolrurl/solr'),
('CKAN_SITE_ID', 'my-site'),
('CKAN_DB', 'postgresql://mydeprectatesqlurl/'),
('CKAN_SMTP_SERVER', 'mail.example.com'),
('CKAN_SMTP_STARTTLS', 'True'),
('CKAN_SMTP_USER', 'my_user'),
('CKAN_SMTP_PASSWORD', 'password'),
('CKAN_SMTP_MAIL_FROM', 'server@example.com'),
('CKAN_MAX_UPLOAD_SIZE_MB', '50')
]

def _setup_env_vars(self):
for env_var, value in self.ENV_VAR_LIST:
os.environ.setdefault(env_var, value)
# plugin.load() will force the config to update
p.load()

def setup(self):
self._old_config = dict(config)

def teardown(self):
for env_var, _ in self.ENV_VAR_LIST:
if os.environ.get(env_var, None):
del os.environ[env_var]
config.update(self._old_config)
# plugin.load() will force the config to update
p.load()

def test_update_config_env_vars(self):
'''
Setting an env var from the whitelist will set the appropriate option
in config object.
'''
self._setup_env_vars()

nosetools.assert_equal(config['solr_url'], 'http://mynewsolrurl/solr')
nosetools.assert_equal(config['sqlalchemy.url'],
'postgresql://mynewsqlurl/')
nosetools.assert_equal(config['ckan.datastore.write_url'],
'http://mynewdbwriteurl/')
nosetools.assert_equal(config['ckan.datastore.read_url'],
'http://mynewdbreadurl/')
nosetools.assert_equal(config['ckan.site_id'], 'my-site')
nosetools.assert_equal(config['smtp.server'], 'mail.example.com')
nosetools.assert_equal(config['smtp.starttls'], 'True')
nosetools.assert_equal(config['smtp.user'], 'my_user')
nosetools.assert_equal(config['smtp.password'], 'password')
nosetools.assert_equal(config['smtp.mail_from'], 'server@example.com')
nosetools.assert_equal(config['ckan.max_resource_size'], '50')

def test_update_config_db_url_precedence(self):
'''CKAN_SQLALCHEMY_URL in the env takes precedence over CKAN_DB'''
os.environ.setdefault('CKAN_DB', 'postgresql://mydeprectatesqlurl/')
os.environ.setdefault('CKAN_SQLALCHEMY_URL',
'postgresql://mynewsqlurl/')
p.load()

nosetools.assert_equal(config['sqlalchemy.url'],
'postgresql://mynewsqlurl/')


class TestSiteUrlMandatory(object):

@helpers.change_config('ckan.site_url', '')
def test_missing_siteurl(self):
nosetools.assert_raises(RuntimeError, environment.update_config)

@helpers.change_config('ckan.site_url', 'demo.ckan.org')
def test_siteurl_missing_schema(self):
nosetools.assert_raises(RuntimeError, environment.update_config)

@helpers.change_config('ckan.site_url', 'ftp://demo.ckan.org')
def test_siteurl_wrong_schema(self):
nosetools.assert_raises(RuntimeError, environment.update_config)

@helpers.change_config('ckan.site_url', 'http://demo.ckan.org/')
def test_siteurl_removes_backslash(self):
ENV_VAR_LIST = [
(u"CKAN_SQLALCHEMY_URL", u"postgresql://mynewsqlurl/"),
(u"CKAN_DATASTORE_WRITE_URL", u"http://mynewdbwriteurl/"),
(u"CKAN_DATASTORE_READ_URL", u"http://mynewdbreadurl/"),
(u"CKAN_SOLR_URL", u"http://mynewsolrurl/solr"),
(u"CKAN_SITE_ID", u"my-site"),
(u"CKAN_DB", u"postgresql://mydeprectatesqlurl/"),
(u"CKAN_SMTP_SERVER", u"mail.example.com"),
(u"CKAN_SMTP_STARTTLS", u"True"),
(u"CKAN_SMTP_USER", u"my_user"),
(u"CKAN_SMTP_PASSWORD", u"password"),
(u"CKAN_SMTP_MAIL_FROM", u"server@example.com"),
(u"CKAN_MAX_UPLOAD_SIZE_MB", u"50"),
]


@pytest.fixture
def reset_env():
"""Reset all environment variables that were patched during tests.
"""
yield
for env_var, _ in ENV_VAR_LIST:
if os.environ.get(env_var, None):
del os.environ[env_var]
p.load()


@pytest.mark.usefixtures(u"reset_env")
def test_update_config_env_vars(ckan_config):
"""
Setting an env var from the whitelist will set the appropriate option
in config object.
"""
for env_var, value in ENV_VAR_LIST:
os.environ.setdefault(env_var, value)
# plugin.load() will force the config to update
p.load()

assert ckan_config[u"solr_url"] == u"http://mynewsolrurl/solr"
assert ckan_config[u"sqlalchemy.url"] == u"postgresql://mynewsqlurl/"
assert (
ckan_config[u"ckan.datastore.write_url"] == u"http://mynewdbwriteurl/"
)
assert ckan_config[u"ckan.datastore.read_url"] == u"http://mynewdbreadurl/"
assert ckan_config[u"ckan.site_id"] == u"my-site"
assert ckan_config[u"smtp.server"] == u"mail.example.com"
assert ckan_config[u"smtp.starttls"] == u"True"
assert ckan_config[u"smtp.user"] == u"my_user"
assert ckan_config[u"smtp.password"] == u"password"
assert ckan_config[u"smtp.mail_from"] == u"server@example.com"
assert ckan_config[u"ckan.max_resource_size"] == u"50"


@pytest.mark.usefixtures("reset_env")
def test_update_config_db_url_precedence(ckan_config):
"""CKAN_SQLALCHEMY_URL in the env takes precedence over CKAN_DB"""
os.environ.setdefault("CKAN_DB", "postgresql://mydeprectatesqlurl/")
os.environ.setdefault("CKAN_SQLALCHEMY_URL", "postgresql://mynewsqlurl/")
p.load()

assert ckan_config["sqlalchemy.url"] == "postgresql://mynewsqlurl/"


@pytest.mark.ckan_config("ckan.site_url", "")
def test_missing_siteurl():
with pytest.raises(RuntimeError):
environment.update_config()
nosetools.assert_equals(config['ckan.site_url'],
'http://demo.ckan.org')


class TestDisplayTimezone(object):
@pytest.mark.ckan_config("ckan.site_url", "demo.ckan.org")
def test_siteurl_missing_schema():
with pytest.raises(RuntimeError):
environment.update_config()


@pytest.mark.ckan_config("ckan.site_url", "ftp://demo.ckan.org")
def test_siteurl_wrong_schema():
with pytest.raises(RuntimeError):
environment.update_config()

@helpers.change_config('ckan.display_timezone', 'Krypton/Argo City')
def test_missing_timezone(self):
nosetools.assert_raises(CkanConfigurationException, environment.update_config)

@pytest.mark.ckan_config("ckan.site_url", "http://demo.ckan.org/")
def test_siteurl_removes_backslash(ckan_config):
environment.update_config()
assert ckan_config["ckan.site_url"] == "http://demo.ckan.org"


@pytest.mark.ckan_config("ckan.display_timezone", "Krypton/Argo City")
def test_missing_timezone():
with pytest.raises(CkanConfigurationException):
environment.update_config()

0 comments on commit b9df6b6

Please sign in to comment.