Permalink
Browse files

Bug 1118786: Prep deployment and crons for Python 2.7

  • Loading branch information...
pmac committed Jan 7, 2015
1 parent ef6d495 commit e9266ade1f63419aa79a9f0fcc133ccbf0694069
@@ -2,18 +2,33 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import glob
import os
from subprocess import call
import chkcrontab_lib as chkcrontab
from funfactory.settings_base import path
from bedrock.mozorg.tests import TestCase
CRONTAB_FILES = glob.glob(path('etc', 'cron.d', '*'))
CRONTAB_FILE_NAMES = ['bedrock-dev', 'bedrock-stage', 'bedrock-prod']
CRONTAB_FILES = [path('etc', 'cron.d', cf) for cf in CRONTAB_FILE_NAMES]
class TestCrontabFiles(TestCase):
files_generated = False
@classmethod
def setUpClass(cls):
for cron_file in CRONTAB_FILE_NAMES:
call([path('bin', 'gen-crons.py'), '-t', cron_file, '-w', '/path/to/www',
'-s', '/path/to/src'])
@classmethod
def tearDownClass(cls):
for cron_file in CRONTAB_FILE_NAMES:
os.remove(path('etc', 'cron.d', cron_file))
def test_crontab_files_have_newlines(self):
"""Crontab files should end with newline character."""
for filename in CRONTAB_FILES:
@@ -0,0 +1,64 @@
#!/usr/bin/env python
import os
from optparse import OptionParser
from jinja2 import Template
HEADER = '!!AUTO-GENERATED!! Edit {template}.tmpl instead.'
TEMPLATE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'etc', 'cron.d'))
def main():
parser = OptionParser()
parser.add_option('-w', '--webapp',
help='Location of web app (required)')
parser.add_option('-s', '--source',
help='Location of source for the web app (required)')
parser.add_option('-t', '--template',
help='Name of the template (e.g. bedrock-prod)')
parser.add_option('-u', '--user', default='root',
help=('Prefix cron with this user. '
'Only define for cron.d style crontabs.'))
parser.add_option('-p', '--python', default='python2.6',
help='Python interpreter to use.')
(opts, args) = parser.parse_args()
if not opts.webapp:
parser.error('-w must be defined')
if not opts.template:
parser.error('-t must be defined')
django_manage = 'cd %s && %s manage.py' % (opts.webapp, opts.python)
ctx = {
'django_manage': django_manage,
'django_cron': '%s cron' % django_manage,
}
for k, v in ctx.iteritems():
ctx[k] = '%s %s' % (opts.user, v)
# Needs to stay below the opts.user injection.
ctx['user'] = opts.user
ctx['webapp'] = opts.webapp
ctx['source'] = opts.source
ctx['python'] = opts.python
ctx['header'] = HEADER.format(template=opts.template)
tmpl_final_name = os.path.join(TEMPLATE_DIR, opts.template)
tmpl_src_name = tmpl_final_name + '.tmpl'
tmpl_temp_name = tmpl_final_name + '.TEMP'
try:
with open(tmpl_src_name, 'r') as src_fh:
with open(tmpl_temp_name, 'w') as out_fh:
out_fh.write(Template(src_fh.read()).render(**ctx))
except IOError:
parser.error('file must exist: ' + tmpl_src_name)
# atomically move into place
os.rename(tmpl_temp_name, tmpl_final_name)
if __name__ == '__main__':
main()
@@ -7,9 +7,16 @@
import os
import sys
from commander.deploy import task
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_dev_base import * # noqa
@task
def update_cron(ctx):
generate_cron_file(ctx, 'bedrock-dev')
@@ -7,9 +7,16 @@
import os
import sys
from commander.deploy import task
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_base import * # noqa
@task
def update_cron(ctx):
generate_cron_file(ctx, 'bedrock-prod')
@@ -7,9 +7,16 @@
import os
import sys
from commander.deploy import task
# these files are symlinked as 'update.py' in the project root.
ROOT = os.path.dirname(os.path.abspath(__file__))
sys.path.append(ROOT)
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
from deploy_base import * # noqa
@task
def update_cron(ctx):
generate_cron_file(ctx, 'bedrock-stage')
@@ -15,6 +15,7 @@
import commander_settings as settings
PYTHON = getattr(settings, 'PYTHON_PATH', 'python2.6')
NEW_RELIC_API_KEY = getattr(settings, 'NEW_RELIC_API_KEY', None)
NEW_RELIC_APP_ID = getattr(settings, 'NEW_RELIC_APP_ID', None)
NEW_RELIC_URL = 'https://rpm.newrelic.com/deployments.xml'
@@ -24,7 +25,7 @@
def management_cmd(ctx, cmd):
"""Run a Django management command correctly."""
with ctx.lcd(settings.SRC_DIR):
ctx.local('LANG=en_US.UTF-8 python2.6 manage.py ' + cmd)
ctx.local('LANG=en_US.UTF-8 {0} manage.py {1}'.format(PYTHON, cmd))
@task
@@ -139,6 +140,8 @@ def update(ctx):
@task
def deploy(ctx):
if 'update_cron' in commands:
commands['update_cron']()
commands['checkin_changes']()
commands['deploy_app']()
commands['ping_newrelic']()
@@ -187,3 +190,12 @@ def generate_desc(from_commit, to_commit, changelog):
else:
desc = get_random_desc()
return desc
def generate_cron_file(ctx, tmpl_name):
with ctx.lcd(settings.SRC_DIR):
ctx.local("{python} bin/gen-crons.py -p {python} -s {src_dir} -w {www_dir} "
"-t {template}".format(python=PYTHON,
src_dir=settings.SRC_DIR,
www_dir=settings.WWW_DIR,
template=tmpl_name))
@@ -10,11 +10,10 @@
@task
def database(ctx):
with ctx.lcd(settings.SRC_DIR):
# only ever run this one on demo and dev.
ctx.local("python2.6 manage.py bedrock_truncate_database --yes-i-am-sure")
ctx.local("python2.6 manage.py syncdb --migrate --noinput")
ctx.local("python2.6 manage.py rnasync")
ctx.local("python2.6 manage.py update_security_advisories --force --quiet")
ctx.local("python2.6 manage.py cron update_reps_ical")
ctx.local("python2.6 manage.py cron update_tweets")
# only ever run this one on demo and dev.
management_cmd(ctx, 'bedrock_truncate_database --yes-i-am-sure')
management_cmd(ctx, 'syncdb --migrate --noinput')
management_cmd(ctx, 'rnasync')
management_cmd(ctx, 'update_security_advisories --force --quiet')
management_cmd(ctx, 'cron update_reps_ical')
management_cmd(ctx, 'cron update_tweets')
@@ -0,0 +1,2 @@
bedrock-*
!bedrock-*.tmpl

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,16 @@
#
# {{ header }}
#
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
0-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org.sh
1-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-django.sh
0 * * * * {{ django_cron }} update_feeds > /dev/null 2>&1
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
*/15 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-svn-locale.sh
# bug 1014586
3 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1

This file was deleted.

Oops, something went wrong.
@@ -0,0 +1,24 @@
#
# {{ header }}
#
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
*/10 * * * * root /data/bedrock/src/update-www.mozilla.org.sh
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
*/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-cron-svn-locale.sh &> /dev/null
# outputs when the run completed to http://www.mozilla.org/media/locale_finished.txt
# disabled per bug 1042074
#1-59/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-product-details.sh
# bug 996144 & 1014586
2 */6 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
# bug 1072999, 1109344
*/30 * * * * {{ django_manage }} update_security_advisories --quiet > /dev/null 2>&1
# bug 1087533
42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
@@ -1,23 +1,23 @@
#
# {{ header }}
#
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
BEDROCK=/data/bedrock-stage/src/www.allizom.org-django/bedrock
BEDROCK_DEPLOY=/data/bedrock-stage/deploy
# TODO: update rest of cron jobs to use the above env variables
# webdev requests that prod auto-updates the PHP side, for similar to old
0-59/10 * * * * root /data/bedrock-stage/src/update-www.allizom.org.sh
*/10 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org.sh
*/15 * * * * root /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
*/15 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
# outputs when the run completed to http://www.allizom.org/media/locale_finished.txt
# In testing for bug 753566 resolution...
# not completely sure how I want to solve this yet
#
# run the crons from the admin node, but on the web nodes... does extra work, but works
#0 * * * * * root echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
*/5 * * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
#0 * * * * * {{ user }} echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
# bug 996144
1 */2 * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
1 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
#
# alternative- run the cron twice right on the admin node, passing --settings=settings/memcache-XXX.py... will this work?
@@ -26,9 +26,9 @@ BEDROCK_DEPLOY=/data/bedrock-stage/deploy
#0 * * * * * root cd /data/www.allizom.org-django/bedrock; python manage.py --settings=settings/memcache-scl3.py cron update_feeds 2>&1 1> /dev/null | grep -v 'old-style Playdoh layout'"
# bug 1072999
41 * * * * root cd $BEDROCK && python manage.py update_security_advisories --quiet
41 * * * * {{ django_manage }} update_security_advisories --quiet
# bug 1087533
# TODO: redirect output to null after initial deployment and verification
42 * * * * root cd $BEDROCK && python manage.py cron update_reps_ical
42 * * * * {{ django_cron }} update_reps_ical

0 comments on commit e9266ad

Please sign in to comment.