Permalink
Browse files

Bug 1118786: Prep deployment and crons for Python 2.7

  • Loading branch information...
1 parent ef6d495 commit e9266ade1f63419aa79a9f0fcc133ccbf0694069 @pmac pmac committed Jan 7, 2015
View
@@ -2,18 +2,33 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-import glob
+import os
+from subprocess import call
import chkcrontab_lib as chkcrontab
from funfactory.settings_base import path
from bedrock.mozorg.tests import TestCase
-CRONTAB_FILES = glob.glob(path('etc', 'cron.d', '*'))
+CRONTAB_FILE_NAMES = ['bedrock-dev', 'bedrock-stage', 'bedrock-prod']
+CRONTAB_FILES = [path('etc', 'cron.d', cf) for cf in CRONTAB_FILE_NAMES]
class TestCrontabFiles(TestCase):
+ files_generated = False
+
+ @classmethod
+ def setUpClass(cls):
+ for cron_file in CRONTAB_FILE_NAMES:
+ call([path('bin', 'gen-crons.py'), '-t', cron_file, '-w', '/path/to/www',
+ '-s', '/path/to/src'])
+
+ @classmethod
+ def tearDownClass(cls):
+ for cron_file in CRONTAB_FILE_NAMES:
+ os.remove(path('etc', 'cron.d', cron_file))
+
def test_crontab_files_have_newlines(self):
"""Crontab files should end with newline character."""
for filename in CRONTAB_FILES:
View
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+import os
+from optparse import OptionParser
+
+from jinja2 import Template
+
+
+HEADER = '!!AUTO-GENERATED!! Edit {template}.tmpl instead.'
+TEMPLATE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'etc', 'cron.d'))
+
+
+def main():
+ parser = OptionParser()
+ parser.add_option('-w', '--webapp',
+ help='Location of web app (required)')
+ parser.add_option('-s', '--source',
+ help='Location of source for the web app (required)')
+ parser.add_option('-t', '--template',
+ help='Name of the template (e.g. bedrock-prod)')
+ parser.add_option('-u', '--user', default='root',
+ help=('Prefix cron with this user. '
+ 'Only define for cron.d style crontabs.'))
+ parser.add_option('-p', '--python', default='python2.6',
+ help='Python interpreter to use.')
+
+ (opts, args) = parser.parse_args()
+
+ if not opts.webapp:
+ parser.error('-w must be defined')
+
+ if not opts.template:
+ parser.error('-t must be defined')
+
+ django_manage = 'cd %s && %s manage.py' % (opts.webapp, opts.python)
+ ctx = {
+ 'django_manage': django_manage,
+ 'django_cron': '%s cron' % django_manage,
+ }
+
+ for k, v in ctx.iteritems():
+ ctx[k] = '%s %s' % (opts.user, v)
+
+ # Needs to stay below the opts.user injection.
+ ctx['user'] = opts.user
+ ctx['webapp'] = opts.webapp
+ ctx['source'] = opts.source
+ ctx['python'] = opts.python
+ ctx['header'] = HEADER.format(template=opts.template)
+
+ tmpl_final_name = os.path.join(TEMPLATE_DIR, opts.template)
+ tmpl_src_name = tmpl_final_name + '.tmpl'
+ tmpl_temp_name = tmpl_final_name + '.TEMP'
+ try:
+ with open(tmpl_src_name, 'r') as src_fh:
+ with open(tmpl_temp_name, 'w') as out_fh:
+ out_fh.write(Template(src_fh.read()).render(**ctx))
+ except IOError:
+ parser.error('file must exist: ' + tmpl_src_name)
+
+ # atomically move into place
+ os.rename(tmpl_temp_name, tmpl_final_name)
+
+if __name__ == '__main__':
+ main()
@@ -7,9 +7,16 @@
import os
import sys
+from commander.deploy import task
+
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_dev_base import * # noqa
+
+
+@task
+def update_cron(ctx):
+ generate_cron_file(ctx, 'bedrock-dev')
@@ -7,9 +7,16 @@
import os
import sys
+from commander.deploy import task
+
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_base import * # noqa
+
+
+@task
+def update_cron(ctx):
+ generate_cron_file(ctx, 'bedrock-prod')
@@ -7,9 +7,16 @@
import os
import sys
+from commander.deploy import task
+
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_base import * # noqa
+
+
+@task
+def update_cron(ctx):
+ generate_cron_file(ctx, 'bedrock-stage')
@@ -15,6 +15,7 @@
import commander_settings as settings
+PYTHON = getattr(settings, 'PYTHON_PATH', 'python2.6')
NEW_RELIC_API_KEY = getattr(settings, 'NEW_RELIC_API_KEY', None)
NEW_RELIC_APP_ID = getattr(settings, 'NEW_RELIC_APP_ID', None)
NEW_RELIC_URL = 'https://rpm.newrelic.com/deployments.xml'
@@ -24,7 +25,7 @@
def management_cmd(ctx, cmd):
"""Run a Django management command correctly."""
with ctx.lcd(settings.SRC_DIR):
- ctx.local('LANG=en_US.UTF-8 python2.6 manage.py ' + cmd)
+ ctx.local('LANG=en_US.UTF-8 {0} manage.py {1}'.format(PYTHON, cmd))
@task
@@ -139,6 +140,8 @@ def update(ctx):
@task
def deploy(ctx):
+ if 'update_cron' in commands:
+ commands['update_cron']()
commands['checkin_changes']()
commands['deploy_app']()
commands['ping_newrelic']()
@@ -187,3 +190,12 @@ def generate_desc(from_commit, to_commit, changelog):
else:
desc = get_random_desc()
return desc
+
+
+def generate_cron_file(ctx, tmpl_name):
+ with ctx.lcd(settings.SRC_DIR):
+ ctx.local("{python} bin/gen-crons.py -p {python} -s {src_dir} -w {www_dir} "
+ "-t {template}".format(python=PYTHON,
+ src_dir=settings.SRC_DIR,
+ www_dir=settings.WWW_DIR,
+ template=tmpl_name))
@@ -10,11 +10,10 @@
@task
def database(ctx):
- with ctx.lcd(settings.SRC_DIR):
- # only ever run this one on demo and dev.
- ctx.local("python2.6 manage.py bedrock_truncate_database --yes-i-am-sure")
- ctx.local("python2.6 manage.py syncdb --migrate --noinput")
- ctx.local("python2.6 manage.py rnasync")
- ctx.local("python2.6 manage.py update_security_advisories --force --quiet")
- ctx.local("python2.6 manage.py cron update_reps_ical")
- ctx.local("python2.6 manage.py cron update_tweets")
+ # only ever run this one on demo and dev.
+ management_cmd(ctx, 'bedrock_truncate_database --yes-i-am-sure')
+ management_cmd(ctx, 'syncdb --migrate --noinput')
+ management_cmd(ctx, 'rnasync')
+ management_cmd(ctx, 'update_security_advisories --force --quiet')
+ management_cmd(ctx, 'cron update_reps_ical')
+ management_cmd(ctx, 'cron update_tweets')
@@ -0,0 +1,2 @@
+bedrock-*
+!bedrock-*.tmpl
@@ -1,10 +0,0 @@
-MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
-0-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org.sh
-1-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-django.sh
-0 * * * * root cd /data/bedrock-dev/src/www-dev.allizom.org-django/bedrock; python manage.py cron update_feeds > /dev/null 2>&1
-*/5 * * * * root cd /data/bedrock-dev/src/www-dev.allizom.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
-
-*/15 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-svn-locale.sh
-
-# bug 1014586
-3 */2 * * * root cd /data/bedrock-dev/src/www-dev.allizom.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
@@ -0,0 +1,16 @@
+#
+# {{ header }}
+#
+
+MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
+
+0-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org.sh
+1-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-django.sh
+0 * * * * {{ django_cron }} update_feeds > /dev/null 2>&1
+*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
+
+*/15 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-svn-locale.sh
+
+# bug 1014586
+3 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
+
@@ -1,27 +0,0 @@
-MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
-BEDROCK=/data/bedrock/src/www.mozilla.org-django/bedrock
-BEDROCK_DEPLOY=/data/bedrock/deploy
-# TODO: update rest of cron jobs to use the above env variables
-
-# webdev requests that prod auto-updates the PHP side, for similar to old
-# setup... not currently auto-updating Django side
-# This will likely change to Freddo / Chief soon
-*/10 * * * * root /data/bedrock/src/update-www.mozilla.org.sh
-
-*/5 * * * * root cd /data/bedrock/src/www.mozilla.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
-*/15 * * * * root /data/bedrock/src/update-www.mozilla.org-cron-svn-locale.sh &> /dev/null
-# outputs when the run completed to http://www.mozilla.org/media/locale_finished.txt
-
-# disabled per bug 1042074
-#1-59/15 * * * * root /data/bedrock/src/update-www.mozilla.org-product-details.sh
-
-# bug 996144 & 1014586
-2 */6 * * * root cd /data/bedrock/src/www.mozilla.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
-
-# bug 1072999, 1109344
-*/30 * * * * root cd /data/bedrock/src/www.mozilla.org-django/bedrock; python manage.py update_security_advisories --quiet > /dev/null 2>&1
-
-# bug 1087533
-# TODO: redirect stdout and stderr to null after initial deployment and verification
-42 * * * * root cd $BEDROCK && python manage.py cron update_reps_ical
-
@@ -0,0 +1,24 @@
+#
+# {{ header }}
+#
+
+MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
+
+*/10 * * * * root /data/bedrock/src/update-www.mozilla.org.sh
+
+*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
+*/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-cron-svn-locale.sh &> /dev/null
+# outputs when the run completed to http://www.mozilla.org/media/locale_finished.txt
+
+# disabled per bug 1042074
+#1-59/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-product-details.sh
+
+# bug 996144 & 1014586
+2 */6 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
+
+# bug 1072999, 1109344
+*/30 * * * * {{ django_manage }} update_security_advisories --quiet > /dev/null 2>&1
+
+# bug 1087533
+42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
+
@@ -1,23 +1,23 @@
+#
+# {{ header }}
+#
+
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
-BEDROCK=/data/bedrock-stage/src/www.allizom.org-django/bedrock
-BEDROCK_DEPLOY=/data/bedrock-stage/deploy
-# TODO: update rest of cron jobs to use the above env variables
-# webdev requests that prod auto-updates the PHP side, for similar to old
-0-59/10 * * * * root /data/bedrock-stage/src/update-www.allizom.org.sh
+*/10 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org.sh
-*/15 * * * * root /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
+*/15 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
# outputs when the run completed to http://www.allizom.org/media/locale_finished.txt
# In testing for bug 753566 resolution...
# not completely sure how I want to solve this yet
#
# run the crons from the admin node, but on the web nodes... does extra work, but works
-#0 * * * * * root echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
-*/5 * * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
+#0 * * * * * {{ user }} echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
+*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
# bug 996144
-1 */2 * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
+1 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
#
# alternative- run the cron twice right on the admin node, passing --settings=settings/memcache-XXX.py... will this work?
@@ -26,9 +26,9 @@ BEDROCK_DEPLOY=/data/bedrock-stage/deploy
#0 * * * * * root cd /data/www.allizom.org-django/bedrock; python manage.py --settings=settings/memcache-scl3.py cron update_feeds 2>&1 1> /dev/null | grep -v 'old-style Playdoh layout'"
# bug 1072999
-41 * * * * root cd $BEDROCK && python manage.py update_security_advisories --quiet
+41 * * * * {{ django_manage }} update_security_advisories --quiet
# bug 1087533
# TODO: redirect output to null after initial deployment and verification
-42 * * * * root cd $BEDROCK && python manage.py cron update_reps_ical
+42 * * * * {{ django_cron }} update_reps_ical

0 comments on commit e9266ad

Please sign in to comment.