Merge branch 'master' into html-emails

.circleci/config.yml

@@ -1,102 +1,146 @@
 version: 2
+defaults:
+  init_environemnt: &init_environment
+    run: |
+      # SOLR config
+      cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
+      service jetty9 restart || true  # erroring out but does seem to work
+
+      # Database Creation
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+      createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+      # Database Initialization
+      ckan -c test-core-circle-ci.ini datastore set-permissions | psql --host=ckan-postgres --username=ckan
+      ckan -c test-core-circle-ci.ini db init
+  install_deps: &install_deps
+    run: |
+      # OS Dependencies
+      apt update
+      case $CIRCLE_NODE_INDEX in
+        $NODE_TESTS_CONTAINER)
+        curl -sL https://deb.nodesource.com/setup_10.x | bash -
+        apt install -y nodejs
+        npm install mocha-phantomjs@3.5.0 phantomjs@~1.9.1
+        ;;
+      esac
+      apt install -y postgresql-client solr-jetty openjdk-8-jdk
+  run_tests: &run_tests
+    # Tests Backend, split across containers by segments
+    run: |
+      mkdir -p ~/junit
+      case $CIRCLE_NODE_INDEX in
+        0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
+        ;;
+        1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
+        ;;
+        2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
+        ;;
+        3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
+        ;;
+      esac
+
+  # Tests Frontend, only in one container
+  start_test_server: &start_test_server
+    run:
+      command: |
+        case $CIRCLE_NODE_INDEX in
+          $NODE_TESTS_CONTAINER) ckan -c test-core-circle-ci.ini run
+          ;;
+        esac
+      background: true
+  run_front_tests: &run_front_tests
+    run:
+      command: |
+        case $CIRCLE_NODE_INDEX in
+          $NODE_TESTS_CONTAINER)
+            sleep 5
+            /root/project/node_modules/mocha-phantomjs/bin/mocha-phantomjs \
+              http://localhost:5000/base/test/index.html
+          ;;
+        esac
+  ckan_env: &ckan_env
+    environment:
+      CKAN_DATASTORE_POSTGRES_DB: datastore_test
+      CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+      CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+      CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+      CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+      CKAN_POSTGRES_DB: ckan_test
+      CKAN_POSTGRES_USER: ckan_default
+      CKAN_POSTGRES_PWD: pass
+      PGPASSWORD: ckan
+      NODE_TESTS_CONTAINER: 2
+      PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
+  pg_image: &pg_image
+    image: postgres:10
+    environment:
+      POSTGRES_USER: ckan
+      POSTGRES_PASSWORD: ckan
+    name: ckan-postgres
+
+  redis_image: &redis_image
+    image: redis:3
+    name: ckan-redis
 jobs:
-  test:
+  test-python-2:
     docker:
       - image: python:2-stretch
-        environment:
-          CKAN_DATASTORE_POSTGRES_DB: datastore_test
-          CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
-          CKAN_DATASTORE_POSTGRES_READ_PWD: pass
-          CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
-          CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
-          CKAN_POSTGRES_DB: ckan_test
-          CKAN_POSTGRES_USER: ckan_default
-          CKAN_POSTGRES_PWD: pass
-          PGPASSWORD: ckan
-          NODE_TESTS_CONTAINER: 2
-          PYTEST_COMMON_OPTIONS: -v --ckan-ini=test-core-circle-ci.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml --test-group-count 4  --test-group-random-seed 1
-      - image: postgres:10
-        environment:
-          POSTGRES_USER: ckan
-          POSTGRES_PASSWORD: ckan
-        name: ckan-postgres
-      - image: redis:3
-        name: ckan-redis
+        <<: *ckan_env
+      - <<: *pg_image
+      - <<: *redis_image
 
     parallelism: 4
 
     steps:
       - checkout
 
+      - <<: *install_deps
       - run: |
-          # OS Dependencies
-          apt update
-          case $CIRCLE_NODE_INDEX in
-            $NODE_TESTS_CONTAINER)
-            curl -sL https://deb.nodesource.com/setup_10.x | bash -
-            apt install -y nodejs
-            npm install mocha-phantomjs@3.5.0 phantomjs@~1.9.1
-            ;;
-          esac
-          apt install -y postgresql-client solr-jetty openjdk-8-jdk
-
           # Python Dependencies
           pip install -r requirement-setuptools.txt
           pip install -r requirements-py2.txt
           pip install -r dev-requirements.txt
           python setup.py develop
+      - <<: *init_environment
+      - <<: *run_tests
+      - store_test_results:
+          path: ~/junit
+      - <<: *start_test_server
+      - <<: *run_front_tests
+  test-python-3:
+    docker:
+      - image: python:3-stretch
+        <<: *ckan_env
+      - <<: *pg_image
+      - <<: *redis_image
 
-          # SOLR config
-          cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
-          service jetty9 restart || true  # erroring out but does seem to work
+    parallelism: 4
 
-          # Database Creation
-          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
-          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
-          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+    steps:
+      - checkout
 
-          # Database Initialization
-          paster datastore -c test-core-circle-ci.ini set-permissions | psql --host=ckan-postgres --username=ckan
-          paster db init -c test-core-circle-ci.ini
+      - <<: *install_deps
+      - run: |
+          # Python Dependencies
+          pip install -r requirement-setuptools.txt
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          python setup.py develop
+      - <<: *init_environment
 
       # Tests Backend, split across containers by segments
-      - run: |
-          mkdir -p ~/junit
-          case $CIRCLE_NODE_INDEX in
-            0) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 1
-            ;;
-            1) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 2
-            ;;
-            2) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 3
-            ;;
-            3) python -m pytest $PYTEST_COMMON_OPTIONS --test-group 4
-            ;;
-          esac
+      - <<: *run_tests
       - store_test_results:
           path: ~/junit
-
-      # Tests Frontend, only in one container
-      - run:
-          command: |
-            case $CIRCLE_NODE_INDEX in
-              $NODE_TESTS_CONTAINER) paster serve test-core-circle-ci.ini
-              ;;
-            esac
-          background: true
-      - run:
-          command: |
-            case $CIRCLE_NODE_INDEX in
-              $NODE_TESTS_CONTAINER)
-                sleep 5
-                /root/project/node_modules/mocha-phantomjs/bin/mocha-phantomjs \
-                  http://localhost:5000/base/test/index.html
-              ;;
-            esac
-
+      - <<: *start_test_server
+      - <<: *run_front_tests
 workflows:
   version: 2
   build_and_test:
     jobs:
-      - test
+      - test-python-2
+      - test-python-3

.gitignore

@@ -7,6 +7,7 @@ syntax: glob
 ckan.egg-info/*
 sandbox/*
 dist
+.mypy_cache
 
 # pylons
 development.ini*
@@ -43,6 +44,7 @@ ckan_deb/DEBIAN/prerm
 
 # node.js
 node_modules/
+package-lock.json
 
 # docker
 contrib/docker/.env

bin/running_stats.py

@@ -15,11 +15,11 @@
         package.delete()
         package_stats.increment('deleted')
     else:
-        package_stats.increment('not deleted')    
+        package_stats.increment('not deleted')
 print package_stats.report()
 > deleted: 30
 > not deleted: 70
-    
+
 from running_stats import StatsList
 package_stats = StatsList()
 for package in packages:
@@ -30,7 +30,7 @@
         package_stats.add('not deleted' package.name)
 print package_stats.report()
 > deleted: 30 pollution-uk, flood-regions, river-quality, ...
-> not deleted: 70 spending-bristol, ... 
+> not deleted: 70 spending-bristol, ...
 
 '''
 
@@ -40,11 +40,11 @@ class StatsCount(dict):
     # {category:count}
     _init_value = 0
     report_value_limit = 150
-    
+
     def _init_category(self, category):
-        if not self.has_key(category):
+        if category not in self:
             self[category] = copy.deepcopy(self._init_value)
-        
+
     def increment(self, category):
         self._init_category(category)
         self[category] += 1
@@ -57,7 +57,7 @@ def report_value(self, category):
 
     def report(self, indent=1):
         lines = []
-        categories = self.keys()
+        categories = list(self.keys())
         categories.sort()
         indent_str = '\t' * indent
         for category in categories:

bin/travis-run-tests

@@ -15,13 +15,14 @@ MOCHA_ERROR=$?
 # We are done so kill ckan
 killall paster
 
-# And finally, run the nosetests
-nosetests --ckan --reset-db --with-pylons=test-core.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext ckan ckanext
+# And finally, run the tests
+PYTEST_OPTIONS: -v --ckan-ini=test-core.ini --cov=ckan --cov=ckanext --junitxml=/root/junit/junit.xml
+python -m pytest $PYTEST_COMMON_OPTIONS
 # Did an error occur?
-NOSE_ERROR=$?
+PYTEST_ERROR=$?
 
 [ "0" -ne "$MOCHA_ERROR" ] && echo MOCHA tests have failed
-[ "0" -ne "$NOSE_ERROR" ] && echo NOSE tests have failed
+[ "0" -ne "$PYTEST_ERROR" ] && echo PYTEST tests have failed
 
 # If an error occurred in our tests make sure travis knows
-exit `expr $MOCHA_ERROR + $NOSE_ERROR`
+exit `expr $MOCHA_ERROR + $PYTEST_ERROR`

ckan/authz.py

@@ -3,7 +3,7 @@
 import functools
 import sys
 
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 from logging import getLogger
 
 import six
@@ -13,7 +13,7 @@
 
 import ckan.plugins as p
 import ckan.model as model
-from ckan.common import OrderedDict, _, c
+from ckan.common import _, c
 
 import ckan.lib.maintain as maintain
 
@@ -385,7 +385,7 @@ def get_user_id_for_username(user_name, allow_none=False):
     try:
         if c.userobj and c.userobj.name == user_name:
             return c.userobj.id
-    except TypeError:
+    except (TypeError, AttributeError):
         # c is not available
         pass
     user = model.User.get(user_name)