Bug 1118786: Prep deployment and crons for Python 2.7

mozilla · Jan 7, 2015 · e9266ade1f63419aa79a9f0fcc133ccbf0694069 · e9266ad
1 parent ef6d495
commit e9266ade1f63419aa79a9f0fcc133ccbf0694069
diff --git a/bedrock/base/tests.py b/bedrock/base/tests.py
@@ -2,18 +2,33 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.

-import glob
+import os
+from subprocess import call

 import chkcrontab_lib as chkcrontab
 from funfactory.settings_base import path

 from bedrock.mozorg.tests import TestCase


-CRONTAB_FILES = glob.glob(path('etc', 'cron.d', '*'))
+CRONTAB_FILE_NAMES = ['bedrock-dev', 'bedrock-stage', 'bedrock-prod']
+CRONTAB_FILES = [path('etc', 'cron.d', cf) for cf in CRONTAB_FILE_NAMES]


 class TestCrontabFiles(TestCase):
+    files_generated = False
+
+    @classmethod
+    def setUpClass(cls):
+        for cron_file in CRONTAB_FILE_NAMES:
+            call([path('bin', 'gen-crons.py'), '-t', cron_file, '-w', '/path/to/www',
+                  '-s', '/path/to/src'])
+
+    @classmethod
+    def tearDownClass(cls):
+        for cron_file in CRONTAB_FILE_NAMES:
+            os.remove(path('etc', 'cron.d', cron_file))
+
    def test_crontab_files_have_newlines(self):
        """Crontab files should end with newline character."""
        for filename in CRONTAB_FILES:
diff --git a/bin/gen-crons.py b/bin/gen-crons.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+import os
+from optparse import OptionParser
+
+from jinja2 import Template
+
+
+HEADER = '!!AUTO-GENERATED!! Edit {template}.tmpl instead.'
+TEMPLATE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'etc', 'cron.d'))
+
+
+def main():
+    parser = OptionParser()
+    parser.add_option('-w', '--webapp',
+                      help='Location of web app (required)')
+    parser.add_option('-s', '--source',
+                      help='Location of source for the web app (required)')
+    parser.add_option('-t', '--template',
+                      help='Name of the template (e.g. bedrock-prod)')
+    parser.add_option('-u', '--user', default='root',
+                      help=('Prefix cron with this user. '
+                            'Only define for cron.d style crontabs.'))
+    parser.add_option('-p', '--python', default='python2.6',
+                      help='Python interpreter to use.')
+
+    (opts, args) = parser.parse_args()
+
+    if not opts.webapp:
+        parser.error('-w must be defined')
+
+    if not opts.template:
+        parser.error('-t must be defined')
+
+    django_manage = 'cd %s && %s manage.py' % (opts.webapp, opts.python)
+    ctx = {
+        'django_manage': django_manage,
+        'django_cron': '%s cron' % django_manage,
+    }
+
+    for k, v in ctx.iteritems():
+        ctx[k] = '%s %s' % (opts.user, v)
+
+    # Needs to stay below the opts.user injection.
+    ctx['user'] = opts.user
+    ctx['webapp'] = opts.webapp
+    ctx['source'] = opts.source
+    ctx['python'] = opts.python
+    ctx['header'] = HEADER.format(template=opts.template)
+
+    tmpl_final_name = os.path.join(TEMPLATE_DIR, opts.template)
+    tmpl_src_name = tmpl_final_name + '.tmpl'
+    tmpl_temp_name = tmpl_final_name + '.TEMP'
+    try:
+        with open(tmpl_src_name, 'r') as src_fh:
+            with open(tmpl_temp_name, 'w') as out_fh:
+                out_fh.write(Template(src_fh.read()).render(**ctx))
+    except IOError:
+        parser.error('file must exist: ' + tmpl_src_name)
+
+    # atomically move into place
+    os.rename(tmpl_temp_name, tmpl_final_name)
+
+if __name__ == '__main__':
+    main()
diff --git a/bin/update/deploy-dev.py b/bin/update/deploy-dev.py
@@ -7,9 +7,16 @@
 import os
 import sys

+from commander.deploy import task
+
 # these files are symlinked as 'update.py' in the project root.
 ROOT = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(ROOT)
 sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))

 from deploy_dev_base import *  # noqa
+
+
+@task
+def update_cron(ctx):
+    generate_cron_file(ctx, 'bedrock-dev')
diff --git a/bin/update/deploy-prod.py b/bin/update/deploy-prod.py
@@ -7,9 +7,16 @@
 import os
 import sys

+from commander.deploy import task
+
 # these files are symlinked as 'update.py' in the project root.
 ROOT = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(ROOT)
 sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))

 from deploy_base import *  # noqa
+
+
+@task
+def update_cron(ctx):
+    generate_cron_file(ctx, 'bedrock-prod')
diff --git a/bin/update/deploy-stage.py b/bin/update/deploy-stage.py
@@ -7,9 +7,16 @@
 import os
 import sys

+from commander.deploy import task
+
 # these files are symlinked as 'update.py' in the project root.
 ROOT = os.path.dirname(os.path.abspath(__file__))
 sys.path.append(ROOT)
 sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))

 from deploy_base import *  # noqa
+
+
+@task
+def update_cron(ctx):
+    generate_cron_file(ctx, 'bedrock-stage')
diff --git a/bin/update/deploy_base.py b/bin/update/deploy_base.py
@@ -15,6 +15,7 @@
 import commander_settings as settings


+PYTHON = getattr(settings, 'PYTHON_PATH', 'python2.6')
 NEW_RELIC_API_KEY = getattr(settings, 'NEW_RELIC_API_KEY', None)
 NEW_RELIC_APP_ID = getattr(settings, 'NEW_RELIC_APP_ID', None)
 NEW_RELIC_URL = 'https://rpm.newrelic.com/deployments.xml'
@@ -24,7 +25,7 @@
 def management_cmd(ctx, cmd):
    """Run a Django management command correctly."""
    with ctx.lcd(settings.SRC_DIR):
-        ctx.local('LANG=en_US.UTF-8 python2.6 manage.py ' + cmd)
+        ctx.local('LANG=en_US.UTF-8 {0} manage.py {1}'.format(PYTHON, cmd))


 @task
@@ -139,6 +140,8 @@ def update(ctx):

 @task
 def deploy(ctx):
+    if 'update_cron' in commands:
+        commands['update_cron']()
    commands['checkin_changes']()
    commands['deploy_app']()
    commands['ping_newrelic']()
@@ -187,3 +190,12 @@ def generate_desc(from_commit, to_commit, changelog):
        else:
            desc = get_random_desc()
    return desc
+
+
+def generate_cron_file(ctx, tmpl_name):
+    with ctx.lcd(settings.SRC_DIR):
+        ctx.local("{python} bin/gen-crons.py -p {python} -s {src_dir} -w {www_dir} "
+                  "-t {template}".format(python=PYTHON,
+                                         src_dir=settings.SRC_DIR,
+                                         www_dir=settings.WWW_DIR,
+                                         template=tmpl_name))
diff --git a/bin/update/deploy_dev_base.py b/bin/update/deploy_dev_base.py
@@ -10,11 +10,10 @@

 @task
 def database(ctx):
-    with ctx.lcd(settings.SRC_DIR):
-        # only ever run this one on demo and dev.
-        ctx.local("python2.6 manage.py bedrock_truncate_database --yes-i-am-sure")
-        ctx.local("python2.6 manage.py syncdb --migrate --noinput")
-        ctx.local("python2.6 manage.py rnasync")
-        ctx.local("python2.6 manage.py update_security_advisories --force --quiet")
-        ctx.local("python2.6 manage.py cron update_reps_ical")
-        ctx.local("python2.6 manage.py cron update_tweets")
+    # only ever run this one on demo and dev.
+    management_cmd(ctx, 'bedrock_truncate_database --yes-i-am-sure')
+    management_cmd(ctx, 'syncdb --migrate --noinput')
+    management_cmd(ctx, 'rnasync')
+    management_cmd(ctx, 'update_security_advisories --force --quiet')
+    management_cmd(ctx, 'cron update_reps_ical')
+    management_cmd(ctx, 'cron update_tweets')
diff --git a/etc/cron.d/.gitignore b/etc/cron.d/.gitignore
@@ -0,0 +1,2 @@
+bedrock-*
+!bedrock-*.tmpl
diff --git a/etc/cron.d/bedrock-dev b/etc/cron.d/bedrock-dev
diff --git a/etc/cron.d/bedrock-dev.tmpl b/etc/cron.d/bedrock-dev.tmpl
@@ -0,0 +1,16 @@
+#
+# {{ header }}
+#
+
+MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
+
+0-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org.sh
+1-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-django.sh
+0       * * * * {{ django_cron }} update_feeds > /dev/null 2>&1
+*/5    * * * * {{ django_manage }} rnasync > /dev/null 2>&1
+
+*/15 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-svn-locale.sh
+
+# bug 1014586
+3 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
+
diff --git a/etc/cron.d/bedrock-prod b/etc/cron.d/bedrock-prod
diff --git a/etc/cron.d/bedrock-prod.tmpl b/etc/cron.d/bedrock-prod.tmpl
@@ -0,0 +1,24 @@
+#
+# {{ header }}
+#
+
+MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
+
+*/10 * * * * root /data/bedrock/src/update-www.mozilla.org.sh
+
+*/5    * * * * {{ django_manage }} rnasync > /dev/null 2>&1
+*/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-cron-svn-locale.sh &> /dev/null
+# outputs when the run completed to http://www.mozilla.org/media/locale_finished.txt
+
+# disabled per bug 1042074
+#1-59/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-product-details.sh
+
+# bug 996144 & 1014586
+2 */6 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
+
+# bug 1072999, 1109344
+*/30 * * * * {{ django_manage }} update_security_advisories --quiet > /dev/null 2>&1
+
+# bug 1087533
+42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
+
diff --git a/etc/cron.d/bedrock-stage → etc/cron.d/bedrock-stage.tmpl b/etc/cron.d/bedrock-stage → etc/cron.d/bedrock-stage.tmpl
@@ -1,23 +1,23 @@
+#
+# {{ header }}
+#
+
 MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
-BEDROCK=/data/bedrock-stage/src/www.allizom.org-django/bedrock
-BEDROCK_DEPLOY=/data/bedrock-stage/deploy
-# TODO: update rest of cron jobs to use the above env variables

-# webdev requests that prod auto-updates the PHP side, for similar to old
-0-59/10 * * * * root /data/bedrock-stage/src/update-www.allizom.org.sh
+*/10 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org.sh

-*/15 * * * * root /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
+*/15 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
 # outputs when the run completed to http://www.allizom.org/media/locale_finished.txt

 # In testing for bug 753566 resolution...
 # not completely sure how I want to solve this yet
 #
 # run the crons from the admin node, but on the web nodes... does extra work, but works
-#0 * * * * *  root echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
-*/5    * * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
+#0 * * * * *  {{ user }} echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
+*/5    * * * * {{ django_manage }} rnasync > /dev/null 2>&1

 # bug 996144
-1 */2 * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
+1 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1

 #
 # alternative- run the cron twice right on the admin node, passing --settings=settings/memcache-XXX.py... will this work?
@@ -26,9 +26,9 @@ BEDROCK_DEPLOY=/data/bedrock-stage/deploy
 #0 * * * * * root cd /data/www.allizom.org-django/bedrock; python manage.py --settings=settings/memcache-scl3.py cron update_feeds 2>&1 1> /dev/null | grep -v 'old-style Playdoh layout'"

 # bug 1072999
-41 * * * * root cd $BEDROCK && python manage.py update_security_advisories --quiet
+41 * * * * {{ django_manage }} update_security_advisories --quiet

 # bug 1087533
 # TODO: redirect output to null after initial deployment and verification
-42 * * * * root cd $BEDROCK && python manage.py cron update_reps_ical
+42 * * * * {{ django_cron }} update_reps_ical