Python 3 support (#2139)

* Run modernize print syntax fixer

* Copy StringIO imports from carbon.util

* Run python-modernize urllib_six fixer

* Add six to project requirements

* Manually fix a couple of urllib imports

* Run python-modernize imports_six fixer

* Manual fix for slots

* Python 3 locations of izip and izip_longest

* Another import of StringIO

* These email.mime.* imports are the documented names even in Python 2

* Run 2to3 except syntax fixer

* Run python-modernize relative import fixer

* Make floor-ed numbers floats, like Python 2

* Compare byte strings for errors

* Missing missing args test for Python 3

* Make a real list for test

* Import reduce() function from Python 3

* Test response content against binary strings

* Make SafeUnpickler work in Python 3

* Compare more response contents against byte strings

* exc.message is already deprecated in Python 2, use str() instead

* Test response content against byte string

* Fix base64 encoding on Python 3

* Unbound methods in Python 3 are plain functions

* Run python-modernize dict_six fixer, plus some manual tweaks

* Compare another HTTP response content with a bytes string

* Run python-modernize xrange_six fixer, plus some manual tweaks

* Rich comparison methods for Interval class

* Run python-modernize basestring fixer

* Another comparison against a bytes string

* Run python-modernize zip fixer

* Avoid comparing None with integer

* Replace next method with next() function for iterators

* Compare responses as text

* Ensure strings are encoded to bytes before md5 hashing

* Write binary data to binary file

* Use integer division when constructing ranges

* Avoid comparing None with integer

* Integer division to find indices in list

* Fix isinstance(x, unicode) checks

* Another comparison against a byte string

* BufferedHTTPReader processes bytes

* Bytes strings for HTTPResponse body

* Integer division for various list manipulations

* Process another HTTP response as bytes

* Import reduce from functools

* Make a list from the map() call

* Replace compare functions with key functions

* Avoid comparing None with integers

* Integer division for making range

* These sort the other way around

* Fix response parsing and content sorting in metrics test

* Bytes regexes for response contents

* Fix expected HTML pattern for Python 3

* Make HTTPResponse objects with bytes data

* Key functions for sorting that can't return None

* Integer division to construct range

* Truncate float to precision matching code being tested

The code being tested - renderViewDygraph(), takes the str() of the
number. Doing the same in the test code makes it match.

* Add Python 3.6 to test configurations

* Actually use Python 3.6 for py36 env in tox

* Build docs with Django 1.x on Py2

* Tidy up imports for linter

* Eliminate assert as required by codacy

* Can we include a Python 3.6 Travis job like this?

* Ugh, yaml

* Fix setup script for Python 3

* Decode responses from redis on Python 3

* Switch cmp function to key function for sorting

* Update classifiers for Python 3 compatibility

* Update pinned requirements to latest versions

* Remove references to simplejson

* Assume cStringIO is available on Python 2

* support pypy, py3.4 & py3.5

* don't reach into ceres internals

* Don't show u string prefixes in events view

* Various code cleanup

* Clarify Python 2 and 3 paths for SafeUnpickler

* Helper function for tests to make JSON bytes

* Use absolute import

* Explain use of StringIO in setup

* Remove unused import

* Don't expect indented JSON for non-pretty tests

* Add Python 3.4, 3.5 to classifiers

.travis.yml

@@ -2,6 +2,24 @@ language: python
 python: 2.7
 sudo: false
 
+matrix:
+  include:
+    - python: pypy
+      env:
+        - TOXENV=pypy-django111-pyparsing2
+    - python: 3.4
+      env:
+        - TOXENV=py34-django111-pyparsing2
+    - python: 3.5
+      env:
+        - TOXENV=py35-django111-pyparsing2
+    - python: 3.6
+      env:
+        - TOXENV=py36-django111-pyparsing2
+    - python: 3.6
+      env:
+        - TOXENV=lint
+
 env:
   - TOXENV=py27-django18-pyparsing2
   - TOXENV=py27-django19-pyparsing2

check-dependencies.py

@@ -100,17 +100,6 @@
   required += 1
 
 
-# Test for a json module
-try:
-  import json
-except ImportError:
-  try:
-    import simplejson
-  except ImportError:
-    sys.stderr.write("[REQUIRED] Unable to import either the 'json' or 'simplejson' module, at least one is required.\n")
-    required += 1
-
-
 # Test for python-memcached
 try:
   import memcache

contrib/memcache_whisper.py

@@ -336,7 +336,7 @@ def update_many(path,points):
   header = __readHeader(fh)
   now = int( time.time() )
   archives = iter( header['archives'] )
-  currentArchive = archives.next()
+  currentArchive = next(archives)
   #debug('  update_many currentArchive=%s' % str(currentArchive))
   currentPoints = []
   for point in points:
@@ -349,7 +349,7 @@ def update_many(path,points):
         __archive_update_many(fh,header,currentArchive,currentPoints)
         currentPoints = []
       try:
-        currentArchive = archives.next()
+        currentArchive = next(archives)
         #debug('  update_many using next archive %s' % str(currentArchive))
       except StopIteration:
         #debug('  update_many no more archives!')

docs/install.rst

@@ -209,7 +209,6 @@ Despair Not!  Even though running Graphite on Windows is completely unsupported
 .. _python-sqlite2: https://github.com/ghaering/pysqlite
 .. _pytz: https://pypi.python.org/pypi/pytz/
 .. _scandir: https://pypi.python.org/pypi/scandir
-.. _simplejson: http://simplejson.readthedocs.io/
 .. _txAMQP: https://launchpad.net/txamqp/
 .. _uWSGI: http://uwsgi-docs.readthedocs.io/
-.. _docker_repo: https://github.com/graphite-project/docker-graphite-statsd
+.. _docker_repo: https://github.com/graphite-project/docker-graphite-statsd

requirements.txt

@@ -38,10 +38,9 @@
 #
 
 Django>=1.8,<1.11.99
-python-memcached==1.47
-txAMQP==0.4
-simplejson==2.1.6
-django-tagging==0.4.3
+python-memcached==1.58
+txAMQP==0.7
+django-tagging==0.4.6
 gunicorn
 pytz
 pyparsing
@@ -52,3 +51,4 @@ git+git://github.com/graphite-project/whisper.git#egg=whisper
 whitenoise
 scandir
 urllib3
+six

setup.py

@@ -3,15 +3,20 @@
 from __future__ import with_statement
 
 import os
-import ConfigParser
+try:
+    from ConfigParser import ConfigParser, DuplicateSectionError  # Python 2
+except ImportError:
+    from configparser import ConfigParser, DuplicateSectionError  # Python 3
 
 from glob import glob
 from collections import defaultdict
 
+# io.StringIO is strictly unicode only. Python 2 StringIO.StringIO accepts
+# bytes, so we'll conveniently ignore decoding and reencoding the file there.
 try:
-    from io import BytesIO
+    from StringIO import StringIO  # Python 2
 except ImportError:
-    from StringIO import StringIO as BytesIO
+    from io import StringIO  # Python 3
 
 # Graphite historically has an install prefix set in setup.cfg. Being in a
 # configuration file, it's not easy to override it or unset it (for installing
@@ -24,22 +29,22 @@
 # ``setup.cfg``.
 with open('setup.cfg', 'r') as f:
     orig_setup_cfg = f.read()
-cf = ConfigParser.ConfigParser()
-cf.readfp(BytesIO(orig_setup_cfg), 'setup.cfg')
+cf = ConfigParser()
+cf.readfp(StringIO(orig_setup_cfg), 'setup.cfg')
 
 if os.environ.get('GRAPHITE_NO_PREFIX') or os.environ.get('READTHEDOCS'):
     cf.remove_section('install')
 else:
     try:
         cf.add_section('install')
-    except ConfigParser.DuplicateSectionError:
+    except DuplicateSectionError:
         pass
     if not cf.has_option('install', 'prefix'):
         cf.set('install', 'prefix', '/opt/graphite')
     if not cf.has_option('install', 'install-lib'):
         cf.set('install', 'install-lib', '%(prefix)s/webapp')
 
-with open('setup.cfg', 'wb') as f:
+with open('setup.cfg', 'w') as f:
     cf.write(f)
 
 if os.environ.get('USE_SETUPTOOLS'):
@@ -101,16 +106,19 @@
       package_data={'graphite' :
         ['templates/*', 'local_settings.py.example']},
       scripts=glob('bin/*'),
-      data_files=webapp_content.items() + storage_dirs + conf_files + examples,
-      install_requires=['Django>=1.8,<1.11.99', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir'],
+      data_files=list(webapp_content.items()) + storage_dirs + conf_files + examples,
+      install_requires=['Django>=1.8,<1.11.99', 'django-tagging==0.4.3', 'pytz', 'pyparsing', 'cairocffi', 'urllib3', 'scandir', 'six'],
       classifiers=[
           'Intended Audience :: Developers',
           'Natural Language :: English',
           'License :: OSI Approved :: Apache Software License',
           'Programming Language :: Python',
           'Programming Language :: Python :: 2',
           'Programming Language :: Python :: 2.7',
-          'Programming Language :: Python :: 2 :: Only',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
           ],
       **setup_kwargs
     )

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist =
-	py27-django1{8,9,10,11}-pyparsing2{,-mysql,-postgresql,-rrdtool,-msgpack},
+	py{27,34,35,36,py}-django1{8,9,10,11}-pyparsing2{,-mysql,-postgresql,-rrdtool,-msgpack},
 	lint, docs
 
 [testenv]
@@ -13,7 +13,6 @@ setenv =
 	mysql: TEST_MYSQL=true
 	postgresql: TEST_POSTGRESQL=true
 passenv = TEST_MYSQL_* TEST_POSTGRESQL_*
-basepython = python2.7
 changedir = webapp
 commands =
 	coverage run --branch --include=graphite/* manage.py test
@@ -49,7 +48,7 @@ deps =
 	pytz
 	git+git://github.com/graphite-project/whisper.git#egg=whisper
 	git+git://github.com/graphite-project/ceres.git#egg=ceres
-	Django
+	Django<2.0
 	pyparsing
 	Sphinx<1.4
 	sphinx_rtd_theme

webapp/graphite/browser/views.py

@@ -24,8 +24,7 @@
 from graphite.util import json
 from graphite.logger import log
 from hashlib import md5
-from urlparse import urlparse, parse_qsl
-from urllib import urlencode
+from six.moves.urllib.parse import urlencode, urlparse, parse_qsl
 
 
 def header(request):

webapp/graphite/carbonlink.py

@@ -12,7 +12,7 @@
 
 
 try:
-  import cPickle as pickle
+  import six.moves.cPickle as pickle
 except ImportError:
   import pickle
 
@@ -134,7 +134,7 @@ def send_request(self, request):
     try:
       conn.sendall(request_packet)
       result = self.recv_response(conn)
-    except Exception,e:
+    except Exception as e:
       self.last_failure[host] = time.time()
       log.cache("Exception getting data from cache %s: %s" % (str(host), e))
     else:
@@ -159,7 +159,7 @@ def send_request_to_all(self, request):
       try:
         conn.sendall(request_packet)
         result = self.recv_response(conn)
-      except Exception,e:
+      except Exception as e:
         self.last_failure[host] = time.time()
         log.cache("Exception getting data from cache %s: %s" % (str(host), e))
       else:

webapp/graphite/compat.py

@@ -1,9 +1,9 @@
-import json
 
 from django import VERSION
 from django.core.serializers.json import DjangoJSONEncoder
 from django.http import (HttpResponse as BaseHttpResponse,
                          HttpResponseBadRequest as Base400)
+from graphite.util import json
 
 
 class ContentTypeMixin(object):

webapp/graphite/composer/views.py

@@ -15,11 +15,11 @@
 import os
 from smtplib import SMTP
 from socket import gethostname
-from email.MIMEMultipart import MIMEMultipart
-from email.MIMEText import MIMEText
-from email.MIMEImage import MIMEImage
-from httplib import HTTPConnection
-from urlparse import urlsplit
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from six.moves.http_client import HTTPConnection
+from six.moves.urllib.parse import urlsplit
 from time import ctime, strftime
 from traceback import format_exc
 from graphite.user_util import getProfile

webapp/graphite/dashboard/models.py

@@ -1,6 +1,7 @@
 from django.db import models
 from graphite.account.models import Profile
 from graphite.util import json
+import six
 
 class Dashboard(models.Model):
   name = models.CharField(primary_key=True, max_length=128)
@@ -22,7 +23,7 @@ def loadState(self, val):
   def setState(self, state, key):
     #XXX Might not need this
     def replace_string(s):
-      if isinstance(s, unicode):
+      if isinstance(s, six.text_type):
         s = s.replace(key, '__VALUE__')
       return s
 

webapp/graphite/dashboard/views.py

@@ -1,10 +1,9 @@
-import json
 import re
 import errno
 
 from os.path import getmtime
-from urllib import urlencode
-from ConfigParser import ConfigParser
+from six.moves.urllib.parse import urlencode
+from six.moves.configparser import ConfigParser
 from django.shortcuts import render_to_response
 from django.http import QueryDict
 from django.conf import settings
@@ -13,9 +12,9 @@
 from django.utils.safestring import mark_safe
 from graphite.compat import HttpResponse
 from graphite.dashboard.models import Dashboard, Template
+from graphite.dashboard.send_graph import send_graph_email
 from graphite.render.views import renderView
-from send_graph import send_graph_email
-
+from graphite.util import json
 
 fieldRegex = re.compile(r'<([^>]+)>')
 defaultScheme = {
@@ -159,7 +158,7 @@ def template(request, name, val):
 
   try:
     config.check()
-  except OSError, e:
+  except OSError as e:
     if e.errno == errno.ENOENT:
       template_conf_missing = True
     else:

webapp/graphite/events/views.py

@@ -1,4 +1,5 @@
 import datetime
+import six
 
 try:
     from django.contrib.sites.requests import RequestSite
@@ -59,7 +60,7 @@ def post_event(request):
         if tags is not None:
             if isinstance(tags, list):
                 tags = ' '.join(tags)
-            elif not isinstance(tags, basestring):
+            elif not isinstance(tags, six.string_types):
                 return HttpResponse(
                     json.dumps({'error': '"tags" must be an array or space-separated string'}),
                     status=400)

webapp/graphite/finders/remote.py

@@ -1,8 +1,7 @@
 import codecs
 import time
 
-from urllib import urlencode
-from urlparse import urlsplit, parse_qs
+from six.moves.urllib.parse import urlencode, urlsplit, parse_qs
 
 from django.conf import settings
 from django.core.cache import cache

webapp/graphite/intervals.py

@@ -89,16 +89,34 @@ def __init__(self, start, end):
   def __eq__(self, other):
     return self.tuple == other.tuple
 
+  def __ne__(self, other):
+    return self.tuple != other.tuple
+
   def __hash__(self):
     return hash( self.tuple )
 
+  def __lt__(self, other):
+    return self.start < self.start
+
+  def __le__(self, other):
+    return self.start <= self.start
+
+  def __gt__(self, other):
+    return self.start > self.start
+
+  def __ge__(self, other):
+    return self.start >= self.start
+
   def __cmp__(self, other):
     return cmp(self.start, other.start)
 
   def __len__(self):
     raise TypeError("len() doesn't support infinite values, use the 'size' attribute instead")
 
-  def __nonzero__(self):
+  def __nonzero__(self):  # Python 2
+    return self.size != 0
+
+  def __bool__(self):  # Python 3
     return self.size != 0
 
   def __repr__(self):

webapp/graphite/metrics/views.py

@@ -11,8 +11,10 @@
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License."""
+
+from functools import reduce
 import pytz
-import urllib
+from six.moves.urllib.parse import unquote_plus
 
 from datetime import datetime
 from django.conf import settings
@@ -290,7 +292,7 @@ def tree_json(nodes, base_path, wildcards=False):
 
     found.add(node.name)
     resultNode = {
-      'text' : urllib.unquote_plus(str(node.name)),
+      'text' : unquote_plus(str(node.name)),
       'id' : base_path + str(node.name),
     }
 

webapp/graphite/node.py

@@ -18,7 +18,7 @@ class BranchNode(Node):
 
 
 class LeafNode(Node):
-  __slots__ = ('reader', 'intervals')
+  __slots__ = ('reader', )
 
   def __init__(self, path, reader):
     Node.__init__(self, path)