Add requirements.txt for python 3

Also: * Install-by-source instructions is for python 3 * Adds initial python2->3 upgrade instructions * Initial adjustments to setup.py and ckan/common.py for import in python3 I'm keen to do 'trunk based development' for python 3 work now. So python 3 is the default in the docs and requirements.txt. I realize that the devs who use master for other work might be a bit surprised/disrupted by these changes. However I think it is worth it because the focus now needs to be python3, and it's better to switch master's default now rather than just before release.
ckan · Nov 8, 2019 · 44ef66c · 44ef66c
1 parent 59006ac
commit 44ef66c
Show file tree

Hide file tree

Showing 8 changed files with 203 additions and 59 deletions.
diff --git a/ckan/common.py b/ckan/common.py
@@ -11,17 +11,18 @@
 from collections import MutableMapping
 
 import flask
-import pylons
 import six
 
 from werkzeug.local import Local, LocalProxy
 
 from flask_babel import (gettext as flask_ugettext,
                          ngettext as flask_ungettext)
-from pylons.i18n import (ugettext as pylons_ugettext,
-                         ungettext as pylons_ungettext)
 
-from pylons import response
+if six.PY2:
+    import pylons
+    from pylons.i18n import (ugettext as pylons_ugettext,
+                            ungettext as pylons_ungettext)
+    from pylons import response
 
 import simplejson as json
 
@@ -38,6 +39,8 @@ def is_flask_request():
     A centralized way to determine whether we are in the context of a
     request being served by Flask or Pylons
     '''
+    if six.PY3:
+        return True
     try:
         pylons.request.environ
         pylons_request_available = True

diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst
@@ -22,7 +22,14 @@ work on CKAN.
 If you're using a Debian-based operating system (such as Ubuntu) install the
 required packages with this command::
 
-    sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server
+    sudo apt-get install python3-dev postgresql libpq-dev git-core solr-jetty openjdk-8-jdk redis-server
+
+.. note::
+
+    For Python 2 (deprecated, but compatible with CKAN 2.9 and earlier), do
+    this instead:
+
+        sudo apt-get install python-dev postgresql libpq-dev python-pip python-virtualenv git-core solr-jetty openjdk-8-jdk redis-server
 
 If you're not using a Debian-based operating system, find the best way to
 install the following packages on your operating system (see
@@ -32,16 +39,15 @@ wiki page for help):
 =====================  ===============================================
 Package                Description
 =====================  ===============================================
-Python                 `The Python programming language, v2.7 <http://www.python.org/getit/>`_
-|postgres|             `The PostgreSQL database system, v9.3 or newer <http://www.postgresql.org/download/>`_
+Python                 `The Python programming language, v3.6 or newer (or v2.7) <https://www.python.org/getit/>`_
+|postgres|             `The PostgreSQL database system, v9.3 or newer <https://www.postgresql.org/docs/9.5/libpq.html>`_
 libpq                  `The C programmer's interface to PostgreSQL <http://www.postgresql.org/docs/8.1/static/libpq.html>`_
-pip                    `A tool for installing and managing Python packages <http://www.pip-installer.org>`_
-virtualenv             `The virtual Python environment builder <http://www.virtualenv.org>`_
-Git                    `A distributed version control system <http://book.git-scm.com/2_installing_git.html>`_
-Apache Solr            `A search platform <http://lucene.apache.org/solr>`_
-Jetty                  `An HTTP server <http://www.eclipse.org/jetty/>`_ (used for Solr).
-OpenJDK JDK            `The Java Development Kit <http://openjdk.java.net/install/>`_ (used by Jetty)
-Redis                  `An in-memory data structure store <http://redis.io/>`_
+python3-venv           `The Python3 virtual environment builder (or for Python 2 use 'virtualenv' instead) <https://virtualenv.pypa.io/en/latest/>`_
+Git                    `A distributed version control system <https://git-scm.com/book/en/v2/Getting-Started-Installing-Git>`_
+Apache Solr            `A search platform <https://lucene.apache.org/solr/>`_
+Jetty                  `An HTTP server <https://www.eclipse.org/jetty/>`_ (used for Solr).
+OpenJDK JDK            `The Java Development Kit <https://openjdk.java.net/install/>`_ (used by Jetty)
+Redis                  `An in-memory data structure store <https://redis.io/>`_
 =====================  ===============================================
 
 
@@ -66,28 +72,17 @@ Redis                  `An in-memory data structure store <http://redis.io/>`_
      mkdir -p ~/ckan/etc
      sudo ln -s ~/ckan/etc |config_parent_dir|
 
-a. Create a Python `virtual environment <http://www.virtualenv.org>`_
+a. Create a Python `virtual environment <https://virtualenv.pypa.io/en/latest/>`_
    (virtualenv) to install CKAN into, and activate it:
 
    .. parsed-literal::
 
        sudo mkdir -p |virtualenv|
        sudo chown \`whoami\` |virtualenv|
-       virtualenv --no-site-packages |virtualenv|
+       pip install --upgrade pip
+       python3 -m venv |virtualenv|
        |activate|
 
-.. note::
-
-    If your system uses Python3 by default (e.g. Ubuntu 18.04) make sure to create
-    the virtualenv using the Python2.7 executable with the ``--python`` option:
-
-    .. parsed-literal::
-
-        sudo mkdir -p |virtualenv|
-        sudo chown \`whoami\` |virtualenv|
-        virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv|
-        |activate|
-
 .. important::
 
    The final command above activates your virtualenv. The virtualenv has to
@@ -105,6 +100,15 @@ a. Create a Python `virtual environment <http://www.virtualenv.org>`_
 
        |activate|
 
+.. note::
+
+    For Python 2 then replace the `python3 -m venv` command with:
+
+    .. parsed-literal::
+
+        virtualenv --python=/usr/bin/python2.7 --no-site-packages |virtualenv|
+        |activate|
+
 b. Install the recommended ``setuptools`` version:
 
    .. parsed-literal::
@@ -140,6 +144,10 @@ d. Install the Python modules that CKAN requires into your virtualenv:
 
        pip install -r |virtualenv|/src/ckan/requirements.txt
 
+.. note::
+
+    For Python 2 then adjust the filename to: `requirements-py2.txt`
+
 e. Deactivate and reactivate your virtualenv, to make sure you're using the
    virtualenv's copies of commands like ``paster`` rather than any system-wide
    installed copies:

diff --git a/doc/maintaining/upgrading/upgrade-to-python3.rst b/doc/maintaining/upgrading/upgrade-to-python3.rst
@@ -0,0 +1,35 @@
+================================================
+Upgrading a CKAN install from python2 to python3
+================================================
+
+These instructions describe how to upgrade a source install of CKAN 2.9 from
+python2 to python3, which is necessary because python2 is end of life, as of
+31st December 2019.
+
+Preparation
+-----------
+
+* Backup your CKAN source, virtualenv and databases, just in case.
+* Upgrade to CKAN 2.9, if you've not done already.
+
+Upgrade
+-------
+
+You'll probably need to deactivate your existing virtual environment::
+
+    deactivate
+
+The existing setup has the virtual environment here: |virtualenv|
+and the CKAN source code underneath in `/usr/lib/ckan/default/src`. We'll move
+that aside in case we need to roll-back:
+
+   .. parsed-literal::
+
+    sudo mv |virtualenv| /usr/lib/ckan/py2
+
+From this doc: :doc:`/maintaining/installing/install-from-source` you need to
+do these sections:
+
+* 1. Install the required packages
+* 2. Install CKAN into a Python virtual environment
+* 6. Link to who.ini
diff --git a/requirements-py2.in b/requirements-py2.in
@@ -0,0 +1,40 @@
+# The file contains the direct ckan requirements.
+# Use pip-compile to create a requirements.txt file from this
+alembic==1.0.0
+Babel==2.3.4
+bleach==3.0.2
+click==6.7
+fanstatic==0.12
+Flask==1.1.1
+Flask-Babel==0.11.2
+Jinja2==2.10.1
+Markdown==2.6.7
+passlib==1.6.5
+paste==1.7.5.1
+PasteScript==2.0.2
+polib==1.0.7
+psycopg2==2.8.2
+python-magic==0.4.15
+pysolr==3.6.0
+Pylons==0.9.7
+python-dateutil>=1.5.0
+pytz==2016.7
+PyUtilib==5.7.1
+pyyaml  # needed by webassets. latest should be fine.
+repoze.who-friendlyform==1.0.8
+repoze.who==2.3
+requests==2.22.0
+Routes==1.13
+rq==1.0
+simplejson==3.10.0
+sqlalchemy-migrate==0.12.0
+SQLAlchemy==1.3.5
+sqlparse==0.2.2
+tzlocal==1.3
+unicodecsv>=0.9
+webassets==0.12.1
+WebHelpers==1.3
+WebOb==1.0.8
+WebTest==1.4.3  # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8
+werkzeug==0.15.5
+zope.interface==4.3.2
diff --git a/requirements-py2.txt b/requirements-py2.txt
@@ -0,0 +1,66 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file requirements=py2.txt requirements-py2.in
+#
+alembic==1.0.0
+babel==2.3.4
+beaker==1.10.1            # via pylons
+bleach==3.0.2
+certifi==2019.3.9         # via requests
+chardet==3.0.4            # via requests
+click==6.7
+decorator==4.4.0          # via pylons, sqlalchemy-migrate
+fanstatic==0.12
+flask-babel==0.11.2
+Flask==1.1.1
+formencode==1.3.1         # via pylons
+funcsigs==1.0.2           # via beaker
+idna==2.8                 # via requests
+itsdangerous==1.1.0       # via flask
+jinja2==2.10.1
+mako==1.0.9               # via alembic, pylons
+markdown==2.6.7
+markupsafe==1.1.1         # via jinja2, mako, webhelpers
+nose==1.3.7               # via pylons
+passlib==1.6.5
+paste==1.7.5.1
+pastedeploy==2.0.1        # via pastescript, pylons
+pastescript==2.0.2
+pbr==5.2.0                # via sqlalchemy-migrate
+polib==1.0.7
+psycopg2==2.8.2
+pygments==2.3.1           # via weberror
+pylons==0.9.7
+pysolr==3.6.0
+python-dateutil==2.8.0
+python-editor==1.0.4      # via alembic
+python-magic==0.4.15
+pytz==2016.7
+PyUtilib==5.7.1
+pyyaml==5.1
+redis==3.2.1              # via rq
+repoze.lru==0.7           # via routes
+repoze.who-friendlyform==1.0.8
+repoze.who==2.3
+requests==2.22.0
+routes==1.13
+rq==1.0
+simplejson==3.10.0
+six==1.12.0               # via bleach, pastescript, python-dateutil, pyutilib.component.core, sqlalchemy-migrate
+sqlalchemy-migrate==0.12.0
+sqlalchemy==1.3.5
+sqlparse==0.2.2
+tempita==0.5.2            # via pylons, sqlalchemy-migrate, weberror
+tzlocal==1.3
+unicodecsv==0.14.1
+urllib3==1.25.2           # via requests
+webassets==0.12.1
+webencodings==0.5.1       # via bleach
+weberror==0.13.1          # via pylons
+webhelpers==1.3
+webob==1.0.8
+webtest==1.4.3
+werkzeug==0.15.5
+zope.interface==4.3.2
diff --git a/requirements.in b/requirements.in
@@ -1,4 +1,4 @@
-# The file contains the direct ckan requirements.
+# The file contains the direct ckan requirements (python3).
 # Use pip-compile to create a requirements.txt file from this
 alembic==1.0.0
 Babel==2.3.4
@@ -16,12 +16,12 @@ polib==1.0.7
 psycopg2==2.8.2
 python-magic==0.4.15
 pysolr==3.6.0
-Pylons==0.9.7
+# Pylons==0.9.7                    - not python3 compatible
 python-dateutil>=1.5.0
 pytz==2016.7
 PyUtilib==5.7.1
 pyyaml  # needed by webassets. latest should be fine.
-repoze.who-friendlyform==1.0.8
+# repoze.who-friendlyform==1.0.8   - not python3 compatible
 repoze.who==2.3
 requests==2.22.0
 Routes==1.13
@@ -33,8 +33,8 @@ sqlparse==0.2.2
 tzlocal==1.3
 unicodecsv>=0.9
 webassets==0.12.1
-WebHelpers==1.3
-WebOb==1.0.8
-WebTest==1.4.3  # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8
+# WebHelpers==1.3                  - not python3 compatible
+# WebOb==1.0.8                     - pylons dependency
+WebTest==1.4.3
 werkzeug==0.15.5
 zope.interface==4.3.2
diff --git a/requirements.txt b/requirements.txt
@@ -6,61 +6,53 @@
 #
 alembic==1.0.0
 babel==2.3.4
-beaker==1.10.1            # via pylons
 bleach==3.0.2
-certifi==2019.3.9         # via requests
+certifi==2019.9.11        # via requests
 chardet==3.0.4            # via requests
 click==6.7
-decorator==4.4.0          # via pylons, sqlalchemy-migrate
+decorator==4.4.1          # via sqlalchemy-migrate
 fanstatic==0.12
 flask-babel==0.11.2
-Flask==1.1.1
-formencode==1.3.1         # via pylons
-funcsigs==1.0.2           # via beaker
+flask==1.1.1
 idna==2.8                 # via requests
 itsdangerous==1.1.0       # via flask
 jinja2==2.10.1
-mako==1.0.9               # via alembic, pylons
+mako==1.1.0               # via alembic
 markdown==2.6.7
-markupsafe==1.1.1         # via jinja2, mako, webhelpers
-nose==1.3.7               # via pylons
+markupsafe==1.1.1         # via jinja2, mako
+nose==1.3.7               # via pyutilib
 passlib==1.6.5
 paste==1.7.5.1
-pastedeploy==2.0.1        # via pastescript, pylons
+pastedeploy==2.0.1        # via pastescript
 pastescript==2.0.2
-pbr==5.2.0                # via sqlalchemy-migrate
+pbr==5.4.3                # via sqlalchemy-migrate
 polib==1.0.7
 psycopg2==2.8.2
-pygments==2.3.1           # via weberror
-pylons==0.9.7
 pysolr==3.6.0
-python-dateutil==2.8.0
+python-dateutil==2.8.1
 python-editor==1.0.4      # via alembic
 python-magic==0.4.15
 pytz==2016.7
-PyUtilib==5.7.1
-pyyaml==5.1
-redis==3.2.1              # via rq
+pyutilib==5.7.1
+pyyaml==5.1.2
+redis==3.3.11             # via rq
 repoze.lru==0.7           # via routes
-repoze.who-friendlyform==1.0.8
 repoze.who==2.3
 requests==2.22.0
 routes==1.13
 rq==1.0
 simplejson==3.10.0
-six==1.12.0               # via bleach, pastescript, python-dateutil, pyutilib.component.core, sqlalchemy-migrate
+six==1.13.0               # via bleach, pastescript, python-dateutil, pyutilib, sqlalchemy-migrate
 sqlalchemy-migrate==0.12.0
 sqlalchemy==1.3.5
 sqlparse==0.2.2
-tempita==0.5.2            # via pylons, sqlalchemy-migrate, weberror
+tempita==0.5.2            # via sqlalchemy-migrate
 tzlocal==1.3
 unicodecsv==0.14.1
-urllib3==1.25.2           # via requests
+urllib3==1.25.6           # via requests
 webassets==0.12.1
 webencodings==0.5.1       # via bleach
-weberror==0.13.1          # via pylons
-webhelpers==1.3
-webob==1.0.8
+webob==1.0.8              # via fanstatic, repoze.who, webtest
 webtest==1.4.3
 werkzeug==0.15.5
 zope.interface==4.3.2
diff --git a/setup.py b/setup.py
@@ -25,7 +25,7 @@
 #
 
 def parse_version(s):
-    return map(int, s.split('.'))
+    return list(map(int, s.split('.')))
 
 HERE = os.path.dirname(__file__)
 with open(os.path.join(HERE, 'requirement-setuptools.txt')) as f: