Local Development with Vagrant and Ansible

.gitignore

@@ -9,8 +9,10 @@ ENV
 .DS_Store
 build
 deploy/last-update
-logs/*
-celerybeat.pid
 celerybeat-schedule
 .gitignore~
 static/scss/*.css.map
+.vagrant
+venv
+*.retry
+*.pid

README.md

@@ -12,7 +12,31 @@ The partitioning between these modules is not as clean as would be ideal. `payme
 
 regluit was originally developed on Django 1.3 (python 2.7) and currently runs on Django 1.8.
 
-Develop
+Development (Vagrant + Virtualbox)
+-------
+
+The recommended method for local development is to create a virtual machine with [Vagrant](https://www.vagrantup.com/) and [Virtualbox](https://www.virtualbox.org/wiki/Downloads).
+With this method, the only requirements on the host machine are `virtualbox` and `vagrant`.
+Vagrant will use the `ansible-local` provisioner, therefore installing python and ansible on the host machine is not necessary.
+
+__Instructions for Ubuntu 16:__
+1. Install virtualbox: `sudo apt-get install virtualbox`
+2. Install vagrant: `sudo apt-get install vagrant`
+3. Clone the `EbookFoundation/regluit` repository.
+4. Navigate to the base directory of the cloned repo (where `Vagrantfile` is located).
+5. Run `vagrant up` to create the VM, install dependencies, and start necessary services.
+  * Note: This step may take up to 15 minutes to complete.
+6. Once the VM has been created, run `vagrant ssh` to log in to the virtual machine you just created. If provisioning was successful, you should see a success message upon login.
+  * If virtualenv doesn't activate upon login, you can do it manually by running `cd /opt/regluit && source venv/bin/activate`
+7. Within the VM, run `./manage.py runserver 0.0.0.0:8000` to start the Django development server.
+8. On your host machine, open your web browser of choice and navigate to `http://127.0.0.1:8000`
+
+__Instructions for other platforms (Windows/OSX):__
+* Steps are essentially the same, except for the installation of Vagrant and Virtualbox. Refer to each package's documentation for specific installation instructions.
+
+_NOTE:_ If running Windows on your host machine, ensure you are running `vagrant up` from an elevated command prompt, e.g. right click on Command Prompt -> Run As Administrator. 
+
+Development (Host Machine)
 -------
 
 Here are some instructions for setting up regluit for development on

Vagrantfile

@@ -0,0 +1,56 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "ubuntu/xenial64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  config.vm.box_check_update = false
+
+  # Setup specific for local machine
+  config.vm.define "regluit-local", primary: true do |local|
+    # Create a private network
+    local.vm.network "private_network", type: "dhcp"
+    local.vm.hostname = "regluit-local"
+
+    # VirtuaLBox provider settings for running locally with Oracle VirtualBox
+    # --uartmode1 disconnected is necessary to disable serial interface, which
+    # is known to cause issues with Ubuntu 16 VM's
+    local.vm.provider "virtualbox" do |vb|
+      vb.name = "regluit-local"
+      vb.memory = 1024
+      vb.cpus = 2
+      vb.customize [ "modifyvm", :id, "--uartmode1", "disconnected" ]
+    end
+
+  end
+
+  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder ".", "/opt/regluit"
+
+  config.vm.network "forwarded_port", guest: 8000, host: 8000
+
+  # Provision node with Ansible running on the Vagrant host
+  # This requires you have Ansible installed locally
+  # Vagrant autogenerates an ansible inventory file to use
+  config.vm.provision "ansible_local" do |ansible|
+    ansible.playbook = "/opt/regluit/provisioning/setup-regluit.yml"
+    ansible.provisioning_path = "/opt/regluit"
+    ansible.verbose = true
+    ansible.install = true
+  end
+
+  config.vm.post_up_message = "Successfully created regluit-local VM. Run 'vagrant ssh' to log in and start the development server."
+
+end

core/models/__init__.py

@@ -45,7 +45,7 @@
     TRANSACTION_STATUS_FAILED,
     TRANSACTION_STATUS_INCOMPLETE
 )
-from regluit.utils import crypto
+from regluit.utils import encryption as crypto
 from regluit.utils.localdatetime import now, date_today
 
 from regluit.core.parameters import (
@@ -244,8 +244,8 @@ def get_archive(self):
                 return r
             except IOError:
                 logger.error(u'could not open {}'.format(self.url))
-        
-        
+
+
     def ebook(self):
         return self.mock_ebook(self)
 
@@ -364,7 +364,7 @@ def ahead(self):
 
 STATUS_CHOICES = (
     ('INITIALIZED','INITIALIZED'),
-    ('ACTIVE', 'ACTIVE'), 
+    ('ACTIVE', 'ACTIVE'),
     ('SUSPENDED',  'SUSPENDED'),
     ('WITHDRAWN', 'WITHDRAWN'),
     ('SUCCESSFUL', 'SUCCESSFUL'),
@@ -388,7 +388,7 @@ class Campaign(models.Model):
     work = models.ForeignKey("Work", related_name="campaigns", null=False)
     managers = models.ManyToManyField(settings.AUTH_USER_MODEL, related_name="campaigns", null=False)
     # status: INITIALIZED, ACTIVE, SUSPENDED, WITHDRAWN, SUCCESSFUL, UNSUCCESSFUL
-    status = models.CharField( max_length=15, null=True, blank=False, default="INITIALIZED", 
+    status = models.CharField( max_length=15, null=True, blank=False, default="INITIALIZED",
         db_index=True, choices=STATUS_CHOICES)
     type = models.PositiveSmallIntegerField(null=False, default=REWARDS,
                                             choices=((REWARDS, 'Pledge-to-unglue campaign'),
@@ -962,7 +962,7 @@ def add_ask_to_ebfs(self, position=0):
                     new_epub_ebf.save()
                     new_epub_ebf.version = version
                     new_ebfs.append(new_epub_ebf)
-                    
+
                     # now make the mobi file
                     new_mobi_ebf = EbookFile.objects.create(edition=edition, format='mobi', asking=True)
                     new_mobi_ebf.file.save(path_for_file('ebf', None), ContentFile(mobi.convert_to_mobi(new_epub_ebf.file.url)))
@@ -992,7 +992,7 @@ def add_ask_to_ebfs(self, position=0):
                 old_ebf.ebook.deactivate()
             old_ebf.file.delete()
             old_ebf.delete()
-        
+
         for non_asking in self.work.ebookfiles().filter(asking=False, ebook__active=True):
             non_asking.ebook.deactivate()
 
@@ -1003,7 +1003,7 @@ def revert_asks(self):
         format_versions = []
         for ebf in EbookFile.objects.filter(edition__work=self.work).exclude(file='').exclude(ebook=None).order_by('-created'):
             format_version = '{}_{}'.format(ebf.format, ebf.ebook.version_label)
-            if ebf.asking: 
+            if ebf.asking:
                 ebf.ebook.deactivate()
             elif format_version in format_versions:
                 # this ebook file has the wrong "asking"
@@ -1157,7 +1157,7 @@ class UserProfile(models.Model):
     librarything_id = models.CharField(max_length=31, blank=True)
     badges = models.ManyToManyField('Badge', related_name='holders', blank=True)
     kindle_email = models.EmailField(max_length=254, blank=True)
-    
+
     # keep track of work the user adds
     works = models.ManyToManyField('Work', related_name='contributors', blank=True)
 

provisioning/host_vars/regluit-local

@@ -0,0 +1 @@
+

provisioning/roles/regluit_common/defaults/main.yml

@@ -0,0 +1,21 @@
+project_path: "/opt/regluit"
+django_settings_module: "regluit.settings.me"
+virtualenv_name: "venv"
+
+# MySQL
+mysql_db_name: "regluit"
+mysql_db_user: "regluit"
+mysql_db_pass: "password123"
+mysql_db_host: "localhost"
+mysql_db_port: 3306
+
+# Task Broker
+broker_transport: "redis"
+broker_host: "localhost"
+broker_port: 6379
+broker_vhost: "0"
+
+# Common.py defaults
+boxstream_api_key: "012345678901234567890123456789"
+boxstream_api_user: "user"
+dropbox_key: "012345678901234"

provisioning/roles/regluit_common/files/celerybeat

@@ -0,0 +1,154 @@
+#!/bin/bash
+# =========================================================
+#  celerybeat - Starts the Celery periodic task scheduler.
+# =========================================================
+#
+# :Usage: /etc/init.d/celerybeat {start|stop|force-reload|restart|try-restart|status}
+# :Configuration file: /etc/default/celerybeat or /etc/default/celeryd
+#
+# See http://docs.celeryq.org/en/latest/cookbook/daemonizing.html#init-script-celerybeat
+# This file is copied from https://github.com/ask/celery/blob/2.4/contrib/generic-init.d/celerybeat
+
+### BEGIN INIT INFO
+# Provides:          celerybeat
+# Required-Start:    $network $local_fs $remote_fs
+# Required-Stop:     $network $local_fs $remote_fs
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: celery periodic task scheduler
+### END INIT INFO
+
+# Cannot use set -e/bash -e since the kill -0 command will abort
+# abnormally in the absence of a valid process ID.
+#set -e
+
+DEFAULT_PID_FILE="/var/run/celerybeat.pid"
+DEFAULT_LOG_FILE="/var/log/celerybeat.log"
+DEFAULT_LOG_LEVEL="INFO"
+DEFAULT_CELERYBEAT="celerybeat"
+
+# /etc/init.d/ssh: start and stop the celery task worker daemon.
+
+if test -f /etc/default/celeryd; then
+    . /etc/default/celeryd
+fi
+
+if test -f /etc/default/celerybeat; then
+    . /etc/default/celerybeat
+fi
+
+CELERYBEAT=${CELERYBEAT:-$DEFAULT_CELERYBEAT}
+CELERYBEAT_PID_FILE=${CELERYBEAT_PID_FILE:-${CELERYBEAT_PIDFILE:-$DEFAULT_PID_FILE}}
+CELERYBEAT_LOG_FILE=${CELERYBEAT_LOG_FILE:-${CELERYBEAT_LOGFILE:-$DEFAULT_LOG_FILE}}
+CELERYBEAT_LOG_LEVEL=${CELERYBEAT_LOG_LEVEL:-${CELERYBEAT_LOGLEVEL:-$DEFAULT_LOG_LEVEL}}
+
+export CELERY_LOADER
+
+CELERYBEAT_OPTS="$CELERYBEAT_OPTS -f $CELERYBEAT_LOG_FILE -l $CELERYBEAT_LOG_LEVEL"
+
+if [ -n "$2" ]; then
+    CELERYBEAT_OPTS="$CELERYBEAT_OPTS $2"
+fi
+
+CELERYBEAT_LOG_DIR=`dirname $CELERYBEAT_LOG_FILE`
+CELERYBEAT_PID_DIR=`dirname $CELERYBEAT_PID_FILE`
+if [ ! -d "$CELERYBEAT_LOG_DIR" ]; then
+    mkdir -p $CELERYBEAT_LOG_DIR
+fi
+if [ ! -d "$CELERYBEAT_PID_DIR" ]; then
+    mkdir -p $CELERYBEAT_PID_DIR
+fi
+
+# Extra start-stop-daemon options, like user/group.
+if [ -n "$CELERYBEAT_USER" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --uid $CELERYBEAT_USER"
+    chown "$CELERYBEAT_USER" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR
+fi
+if [ -n "$CELERYBEAT_GROUP" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --gid $CELERYBEAT_GROUP"
+    chgrp "$CELERYBEAT_GROUP" $CELERYBEAT_LOG_DIR $CELERYBEAT_PID_DIR
+fi
+
+CELERYBEAT_CHDIR=${CELERYBEAT_CHDIR:-$CELERYD_CHDIR}
+if [ -n "$CELERYBEAT_CHDIR" ]; then
+    DAEMON_OPTS="$DAEMON_OPTS --workdir $CELERYBEAT_CHDIR"
+fi
+
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/sbin"
+
+check_dev_null() {
+    if [ ! -c /dev/null ]; then
+        echo "/dev/null is not a character device!"
+        exit 1
+    fi
+}
+
+wait_pid () {
+    pid=$1
+    forever=1
+    i=0
+    while [ $forever -gt 0 ]; do
+        kill -0 $pid 1>/dev/null 2>&1
+        if [ $? -eq 1 ]; then
+            echo "OK"
+            forever=0
+        else
+            kill -TERM "$pid"
+            i=$((i + 1))
+            if [ $i -gt 60 ]; then
+                echo "ERROR"
+                echo "Timed out while stopping (30s)"
+                forever=0
+            else
+                sleep 0.5
+            fi
+        fi
+    done
+}
+
+
+stop_beat () {
+    echo -n "Stopping celerybeat... "
+    if [ -f "$CELERYBEAT_PID_FILE" ]; then
+        wait_pid $(cat "$CELERYBEAT_PID_FILE")
+    else
+        echo "NOT RUNNING"
+    fi
+}
+
+start_beat () {
+    echo "Starting celerybeat..."
+    if [ -n "$VIRTUALENV" ]; then
+        source $VIRTUALENV/bin/activate
+    fi
+    $CELERYBEAT $CELERYBEAT_OPTS $DAEMON_OPTS --detach \
+                --pidfile="$CELERYBEAT_PID_FILE"
+}
+
+
+
+case "$1" in
+  start)
+    check_dev_null
+    start_beat
+    ;;
+  stop)
+    stop_beat
+    ;;
+  reload|force-reload)
+    echo "Use start+stop"
+    ;;
+  restart)
+    echo "Restarting celery periodic task scheduler"
+    stop_beat
+    check_dev_null
+    start_beat
+    ;;
+
+  *)
+    echo "Usage: /etc/init.d/celerybeat {start|stop|restart}"
+    exit 1
+esac
+
+exit 0