Skip to content
Permalink
Browse files

Migrate to Python3

- used 2to3
- fabfile.py -> tasks.py as it makes more sense to use invoke rather
  than fabric2 tasks (fabric1 is not compatible with Python3)
- use feedfinder2 (a copy of the file that has the timeout feature
  contrary to the one on pypi sadly -- TODO: find a saner solution)
- fix feedpaser version fo allow a more python3-compatible version
- fix remaining incompatibilites missed by 2to3
  • Loading branch information...
tibonihoo committed May 31, 2019
1 parent b950051 commit ec1414dc1ef77a92a3296a4c8f0d1a5ded424055
Showing with 717 additions and 755 deletions.
  1. +0 −131 fabfile.py
  2. +1 −2 requirements-dev.txt
  3. +5 −2 requirements_base.txt
  4. +155 −0 tasks.py
  5. +1 −1 wateronmars/settings.py
  6. +1 −1 wateronmars/urls.py
  7. +1 −1 wom_classification/migrations/0001_initial.py
  8. +5 −5 wom_classification/models.py
  9. +3 −3 wom_classification/tests.py
  10. +1 −1 wom_pebbles/migrations/0001_initial.py
  11. +25 −0 wom_pebbles/migrations/0002_auto_20190531_2338.py
  12. +1 −1 wom_pebbles/models.py
  13. +2 −2 wom_pebbles/tasks.py
  14. +1 −1 wom_pebbles/templatetags/html_sanitizers.py
  15. +10 −10 wom_pebbles/tests.py
  16. +1 −1 wom_river/migrations/0001_initial.py
  17. +20 −0 wom_river/migrations/0002_auto_20190531_2338.py
  18. +4 −4 wom_river/tasks.py
  19. +7 −7 wom_river/tests.py
  20. +32 −32 wom_river/utils/feedfinder.py
  21. +188 −0 wom_river/utils/feedfinder2.py
  22. +14 −14 wom_river/utils/netscape_bookmarks.py
  23. +5 −5 wom_river/utils/read_opml.py
  24. +3 −4 wom_river/utils/test.py
  25. +1 −1 wom_tributary/migrations/0001_initial.py
  26. +25 −0 wom_tributary/migrations/0002_auto_20190531_2338.py
  27. +3 −3 wom_tributary/tasks.py
  28. +0 −341 wom_tributary/tests.py
  29. +8 −18 wom_tributary/utils/tweet_summarizers.py
  30. +3 −3 wom_tributary/utils/twitter_oauth.py
  31. +11 −12 wom_user/forms.py
  32. +1 −1 wom_user/migrations/0001_initial.py
  33. +30 −0 wom_user/migrations/0002_auto_20190531_2338.py
  34. +3 −3 wom_user/models.py
  35. +5 −5 wom_user/tasks.py
  36. +92 −92 wom_user/test_collection.py
  37. +18 −18 wom_user/test_river.py
  38. +10 −10 wom_user/test_tributary.py
  39. +5 −5 wom_user/tests.py
  40. +16 −15 wom_user/views.py

This file was deleted.

@@ -1,5 +1,4 @@
-r requirements.txt
fabric>=1.0,<2.0
fabric>=2.4,<3.0
coverage
python-coveralls
mock
@@ -1,6 +1,9 @@
Django>=1.11.19
Django>=1.11.19,<2.0
South==1.0.1
feedparser==5.1.3
feedparser>=5.2.1,<6.0
beautifulsoup4
granary==1.14
python-dateutil
six # for feedfinder2
requests # for feedfinder2

155 tasks.py
@@ -0,0 +1,155 @@
import os
import shutil
import configparser

from invoke import task
from fabric import Connection

DJANGO_APPS = ["wom_classification", "wom_pebbles", "wom_river", "wom_user", "wom_tributary"]

"""This tasks.py will work with an additional configuration file
where the info about deployment hosts setup is described.
The reason for that is to be able to add tasks.py itself to the
source (because the local commands are useful for devs).
The configuration file is called fabhosts.cfg and formated in INI format as follow:
[targetName]
connection = {userA}@{hostX}
site_dir = ~/path/to/wateronmars_site
virtual_env_dir = ../venv
final_deploy_action = touch /path/to/passenger/tmp/restart.txt
provider = ...
- connection gives user and host info of where the deployment will happen
- virtual_end_dir is relative to site_dir
- final_deploy_action can be ommited
- provider can be ommited OR replace all the others if its value is 'heroku'
There must be as many such sections as there are user+host for deployment.
"""

USER_CONF_FILE = "fabhosts.cfg"
USER_CONF = configparser.ConfigParser()
if not os.path.isfile(USER_CONF_FILE):
raise RuntimeError("A file called {0} is needed (see tasks.py's doc for more info).".format(USER_CONF_FILE))
USER_CONF.read("./fabhosts.cfg")

DEPLOY_TARGETS = USER_CONF.sections()

@task
def serve(c):
c.run("python manage.py migrate")
c.run("python manage.py collectstatic")
c.run("python manage.py runserver")

@task
def test(c):
c.run("coverage run manage.py test {0}".format(" ".join(DJANGO_APPS)))

def deploy_heroku(c):
c.run("git pull --rebase heroku master")
c.run("git push heroku master")
# NOTE: for existing apps running with Django1.4, the first upgrade to
# Django1.11 should fail here and be replaced by a manual:
# "heroku run \"python manage.py migrate --fake\""
c.run("heroku run \"python manage.py migrate\"")

def deploy_on_remote(c, target_config):
with Connection(target_config("connection")) as conn:
site_dir = target_config("site_dir")
run_in_dir = lambda cmd: conn.run("cd {0} && {0}".format(site_dir, cmd))
venv_dir = target_config("virtual_env_dir")
run_in_dir("git pull --rebase origin master")
run_in_dir("source {0}/bin/activate && pip install -r requirements_base.txt".format(venv_dir))
# NOTE: for existing apps running with Django1.4, the first upgrade to
# Django1.11 should fail here and be replaced by a manual:
# "python manage.py migrate --fake" !
run_in_dir("source {0}/bin/activate && python manage.py migrate".format(venv_dir))
run_in_dir("source {0}/bin/activate && python manage.py collectstatic".format(venv_dir))
try:
run_in_dir(target_config("final_deploy_action"))
except configparser.NoOptionError:
pass

@task
def deploy(c, targets=None):
if not targets:
targets = DEPLOY_TARGETS
c.run("git pull --rebase origin master")
c.run("git push origin master")
for target in targets:
target_config = USER_CONF.get(target)
if target_config("provider") == "heroku":
return deploy_heroku()
else:
return deploy_on_remote(c, target_config)

@task
def style_check(c):
c.run("flake8 --ignore=E501,E111,E121 ./ --exclude=utils/,.git,__pycache__")


@task
def test_coverage(c):
c.run('coverage report --omit "/tmp/*,_*,*/venv/*,*/migrations/*"')

@task
def schema_reset(c, app_name):
migration_dir = os.path.join("./",app_name,"migrations")
if os.path.isdir(migration_dir):
print("Cleanup past migrations for {0}".format(app_name))
shutil.rmtree(migration_dir)
print("Initialize the migration data for {0}".format(app_name))
c.run("python manage.py makemigrations {0} --initial".format(app_name))
print("""Don't forget that to cancel previous migrations of the actual db (if any) with:
python manage.py migrate {0} zero""".format(app_name))

@task
def schema_update(c, app_name=None):
app_selection = [app_name] if app_name else DJANGO_APPS
for app in app_selection:
try:
c.run("python manage.py makemigrations {0}".format(app))
except:
if app_name is None:
pass
else:
raise
@task
def db_reset(c):
for app_name in DJANGO_APPS:
schema_reset(app_name)
# Just in case we're using a local sql3 db, remove it for
# proper reset (in other cases, cleaning out the db must be
# done before calling this script)
if not os.path.isfile("./db.sql3"):
print("""Couldn't find any local sql3 db.
WARNING: Make sure to clean any db used by Django before the reset !""")
else:
print("Remove db.sql3")
os.remove("./db.sql3")
print("Setup the base db")
c.run("python manage.py migrate")

@task
def db_update(c):
c.run("python manage.py migrate")

@task
def transl_gen(c, lang):
os.chdir("wom_user")
try:
c.run("python ../manage.py makemessages -l {0}".format(lang))
finally:
os.chdir("..")

@task
def transl_compile(c, lang):
os.chdir("wom_user")
try:
c.run("python ../manage.py compilemessages -l {0}".format(lang))
finally:
os.chdir("..")

@@ -59,7 +59,7 @@
djcelery.setup_loader()
USE_CELERY = True
except ImportError:
print "djcelery not available, no background task possible"
print("djcelery not available, no background task possible")
USE_CELERY = False

# Feel free to force de-activation of celery
@@ -19,7 +19,7 @@
#


import settings
from . import settings
from django.conf.urls import include, url

# Uncomment the next two lines to enable the admin:
@@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11 on 2019-05-19 19:41
from __future__ import unicode_literals


from django.conf import settings
from django.db import migrations, models
@@ -45,7 +45,7 @@ class Tag(models.Model):
name = models.CharField(max_length=TAG_NAME_MAX_LENGTH,
unique=True, db_index=True)

def __unicode__(self):
def __str__(self):
return self.name


@@ -62,8 +62,8 @@ class ClassificationData(models.Model):
object_id = models.PositiveIntegerField()
content_object = GenericForeignKey('content_type', 'object_id')

def __unicode__(self):
return u"%s>%s: %s" % (self.owner.username,
def __str__(self):
return "%s>%s: %s" % (self.owner.username,
self.content_object,
list(t for t in self.tags.all()))

@@ -79,7 +79,7 @@ def get_item_tags(user,item):
return Tag.objects.none()
else:
if numRes>1:
print "WARNING: unexpectedly found more than one CD for %s's%s" % (user,item)
print("WARNING: unexpectedly found more than one CD for %s's%s" % (user,item))
# in any case return the first tags
return qs[0].tags

@@ -111,7 +111,7 @@ def set_item_tags(user,item,tags):
cd.save()
else:
if numRes>1:
print "WARNING: unexpectedly found more than one CD for %s's%s" % (user,item)
print("WARNING: unexpectedly found more than one CD for %s's%s" % (user,item))
cd = qs[0]
cd.tags.add(*tags)
return cd
@@ -36,7 +36,7 @@


if TAG_NAME_MAX_LENGTH>255:
print "WARNING: the current max length for TAG name may cause portability problems (see https://docs.djangoproject.com/en/1.4/ref/databases/#character-fields)"
print("WARNING: the current max length for TAG name may cause portability problems (see https://docs.djangoproject.com/en/1.4/ref/databases/#character-fields)")


class TagModelTest(TestCase):
@@ -83,11 +83,11 @@ def test_construction_for_any_model_instance(self):
self.assertEqual(0,nbUserBCD)

def test_stringification(self):
"""Test the __unicode__ method since it is slightly non-trivial."""
"""Test the __str__ method since it is slightly non-trivial."""
item = User.objects.create(username="ItemC")
ci = ClassificationData.objects.create(owner=self.user_a,
content_object=item)
unicodeStr = unicode(ci)
unicodeStr = str(ci)
self.assertIn("ItemC",unicodeStr)
self.assertIn("UserA",unicodeStr)

0 comments on commit ec1414d

Please sign in to comment.
You can’t perform that action at this time.