Upgrading to python 3

data_scraper/act.py

@@ -1,9 +1,11 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import requests
 import base64
-from urllib import quote
+from urllib.parse import quote
 from yvih import models, db
-from base import BaseData
+from .base import BaseData
 from bs4 import BeautifulSoup
 import csv
 
@@ -16,6 +18,5 @@ def __init__(self):
     def actData(self):
         page = requests.get(self.list_url).content
         soup = BeautifulSoup(page)
-        print soup.prettify()
         # table = soup.find('table', 'tablesorter')
         # print table

data_scraper/base.py

@@ -1,4 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from yvih import app, models, db
 import os
 import requests

data_scraper/federal.py

@@ -1,8 +1,10 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import requests
-from urllib import quote
+from urllib.parse import quote
 from yvih import models, db
-from base import BaseData
+from .base import BaseData
 from bs4 import BeautifulSoup
 import csv
 

data_scraper/qld.py

@@ -1,10 +1,12 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import requests
 import base64
 import re
-from urllib import quote
+from urllib.parse import quote
 from yvih import models, db
-from base import BaseData
+from .base import BaseData
 from bs4 import BeautifulSoup
 import csv
 
@@ -67,7 +69,7 @@ def memberPage(self, link):
                 db.session.add(new_link)
 
         # phone numbers
-        numbers = re.findall(ur'[A-Za-z\:\ ]{5,7}\([0-9]*\)\ [0-9]*\ [0-9]*', bio.text)
+        numbers = re.findall(r'[A-Za-z\:\ ]{5,7}\([0-9]*\)\ [0-9]*\ [0-9]*', bio.text)
         for i, number in enumerate(numbers):
             split_number = number.split(': ')
             if (i > 1 and split_number[0] != 'Fax') or (i > 2):
@@ -84,11 +86,11 @@ def memberPage(self, link):
             if len(contents) == 0:
                 continue
             # get rid of header and any tag elements
-            contents = [content for content in contents[1:] if isinstance(content, unicode)]
+            contents = [content for content in contents[1:] if isinstance(content, str)]
             address_line1 = contents[0]
             address_line2 = contents[1] if len(contents) > 2 else None
             suburb = contents[-1].split(' ')[0]
-            postcode = re.findall(ur'[0-9]{4}', contents[-1])
+            postcode = re.findall(r'[0-9]{4}', contents[-1])
             postcode = postcode[0] if len(postcode) > 0 else None
             if i > 0:
                 address_type = models.AddressType.query.get(6)
@@ -103,11 +105,11 @@ def memberPage(self, link):
             if len(contents) == 0:
                 continue
             # get rid of header and any tag elements
-            contents = [content for content in contents[1:] if isinstance(content, unicode)]
+            contents = [content for content in contents[1:] if isinstance(content, str)]
             address_line1 = contents[0]
             address_line2 = contents[1] if len(contents) > 2 else None
             suburb = contents[-1].split(' ')[0]
-            postcode = re.findall(ur'[0-9]{4}', contents[-1])
+            postcode = re.findall(r'[0-9]{4}', contents[-1])
             postcode = postcode[0] if len(postcode) > 0 else None
             if i > 0:
                 address_type = models.AddressType.query.get(6)

manage.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import unittest
 import os
 import coverage
@@ -28,7 +31,7 @@ def cov():
     unittest.TextTestRunner(verbosity=2).run(tests)
     cov.stop()
     cov.save()
-    print 'Coverage Summary:'
+    print('Coverage Summary:')
     cov.report()
     basedir = os.path.abspath(os.path.dirname(__file__))
     covdir = os.path.join(basedir, 'coverage')

requirements.txt

@@ -4,10 +4,9 @@ Flask-Script==2.0.5
 Flask-Testing==0.4.2
 Jinja2==2.7.3
 MarkupSafe==0.23
-SQLAlchemy==0.9.8
-Werkzeug==0.9.6
+SQLAlchemy==1.0.0
+Werkzeug==0.10.4
 beautifulsoup4==4.3.2
 coverage==3.7.1
 itsdangerous==0.24
-requests==2.5.1
-wsgiref==0.1.2
+requests==2.6.0

run.py

@@ -1,7 +1,7 @@
-from yvih import app
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 
-app.config['DEBUG'] = True
-app.config['PRESERVE_CONTEXT_ON_EXCEPTION'] = False
+from yvih import app
 
 if __name__ == '__main__':
     app.run()

tests/base.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 import unittest
 from flask.ext.testing import TestCase
 from yvih import app, db

tests/home.py

@@ -1,10 +1,13 @@
-from base import BaseTestCase
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from tests.base import BaseTestCase
 
 class HomeTestCase(BaseTestCase):
     def test_home_displays_api(self):
         response = self.client.get('/')
-        self.assertIn('Your Voice in House', response.data)
-        self.assertIn('Search', response.data)
+        self.assertIn('Your Voice in House', response.data.decode('utf-8'))
+        self.assertIn('Search', response.data.decode('utf-8'))
 
     def test_non_get_method(self):
         response = self.client.post('/')

tests/members.py

@@ -1,11 +1,14 @@
-from urllib import quote
-from base import BaseTestCase
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from urllib.parse import quote
+from tests.base import BaseTestCase
 
 class MembersTestCase(BaseTestCase):
 
     def test_member_html(self):
         response = self.client.get('/members/')
-        self.assertIn('Members', response.data)
+        self.assertIn('Members', response.data.decode('utf-8'))
 
     def test_member_json(self):
         headers = [('Accept', 'application/json')]

yvih/chambers/views.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from flask import redirect, jsonify, render_template, request, url_for, Blueprint
 from yvih import db
 from yvih.models import Chamber

yvih/electorates/views.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from flask import redirect, jsonify, render_template, request, url_for, Blueprint
 from yvih import db
 from yvih.models import Electorate

yvih/home/views.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from flask import redirect, render_template, request, url_for, Blueprint
 
 home_blueprint = Blueprint(

yvih/members/views.py

@@ -1,3 +1,6 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from flask import redirect, jsonify, render_template, request, url_for, Blueprint, abort
 from yvih import db, request_wants_json
 from yvih.models import Member
@@ -18,7 +21,7 @@ def members( conditions=None ):
 
         # get query and turn it into a dictionary
         conditions = conditions.split('/')
-        query = zip(conditions[0::2], conditions[1::2])
+        query = list(zip(conditions[0::2], conditions[1::2]))
 
         # check that all fields are valid
         if not parameter_accepted(query):
@@ -56,9 +59,9 @@ def members( conditions=None ):
 def parameter_accepted(query):
     accepted = {
         'id' : int,
-        'first_name' : unicode,
-        'second_name' : unicode,
-        'role' : unicode
+        'first_name' : str,
+        'second_name' : str,
+        'role' : str
     }
     for conditions in query:
         if conditions[0] not in accepted:

yvih/models.py

@@ -1,4 +1,6 @@
-#!/usr/bin/python
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 from yvih import db
 from sqlalchemy import event
 from sqlalchemy.orm import mapper