Skip to content

Commit e9266ad

Browse files
committed
Bug 1118786: Prep deployment and crons for Python 2.7
1 parent ef6d495 commit e9266ad

13 files changed

+175
-59
lines changed

bedrock/base/tests.py

+17-2
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,33 @@
22
# License, v. 2.0. If a copy of the MPL was not distributed with this
33
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
44

5-
import glob
5+
import os
6+
from subprocess import call
67

78
import chkcrontab_lib as chkcrontab
89
from funfactory.settings_base import path
910

1011
from bedrock.mozorg.tests import TestCase
1112

1213

13-
CRONTAB_FILES = glob.glob(path('etc', 'cron.d', '*'))
14+
CRONTAB_FILE_NAMES = ['bedrock-dev', 'bedrock-stage', 'bedrock-prod']
15+
CRONTAB_FILES = [path('etc', 'cron.d', cf) for cf in CRONTAB_FILE_NAMES]
1416

1517

1618
class TestCrontabFiles(TestCase):
19+
files_generated = False
20+
21+
@classmethod
22+
def setUpClass(cls):
23+
for cron_file in CRONTAB_FILE_NAMES:
24+
call([path('bin', 'gen-crons.py'), '-t', cron_file, '-w', '/path/to/www',
25+
'-s', '/path/to/src'])
26+
27+
@classmethod
28+
def tearDownClass(cls):
29+
for cron_file in CRONTAB_FILE_NAMES:
30+
os.remove(path('etc', 'cron.d', cron_file))
31+
1732
def test_crontab_files_have_newlines(self):
1833
"""Crontab files should end with newline character."""
1934
for filename in CRONTAB_FILES:

bin/gen-crons.py

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env python
2+
import os
3+
from optparse import OptionParser
4+
5+
from jinja2 import Template
6+
7+
8+
HEADER = '!!AUTO-GENERATED!! Edit {template}.tmpl instead.'
9+
TEMPLATE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'etc', 'cron.d'))
10+
11+
12+
def main():
13+
parser = OptionParser()
14+
parser.add_option('-w', '--webapp',
15+
help='Location of web app (required)')
16+
parser.add_option('-s', '--source',
17+
help='Location of source for the web app (required)')
18+
parser.add_option('-t', '--template',
19+
help='Name of the template (e.g. bedrock-prod)')
20+
parser.add_option('-u', '--user', default='root',
21+
help=('Prefix cron with this user. '
22+
'Only define for cron.d style crontabs.'))
23+
parser.add_option('-p', '--python', default='python2.6',
24+
help='Python interpreter to use.')
25+
26+
(opts, args) = parser.parse_args()
27+
28+
if not opts.webapp:
29+
parser.error('-w must be defined')
30+
31+
if not opts.template:
32+
parser.error('-t must be defined')
33+
34+
django_manage = 'cd %s && %s manage.py' % (opts.webapp, opts.python)
35+
ctx = {
36+
'django_manage': django_manage,
37+
'django_cron': '%s cron' % django_manage,
38+
}
39+
40+
for k, v in ctx.iteritems():
41+
ctx[k] = '%s %s' % (opts.user, v)
42+
43+
# Needs to stay below the opts.user injection.
44+
ctx['user'] = opts.user
45+
ctx['webapp'] = opts.webapp
46+
ctx['source'] = opts.source
47+
ctx['python'] = opts.python
48+
ctx['header'] = HEADER.format(template=opts.template)
49+
50+
tmpl_final_name = os.path.join(TEMPLATE_DIR, opts.template)
51+
tmpl_src_name = tmpl_final_name + '.tmpl'
52+
tmpl_temp_name = tmpl_final_name + '.TEMP'
53+
try:
54+
with open(tmpl_src_name, 'r') as src_fh:
55+
with open(tmpl_temp_name, 'w') as out_fh:
56+
out_fh.write(Template(src_fh.read()).render(**ctx))
57+
except IOError:
58+
parser.error('file must exist: ' + tmpl_src_name)
59+
60+
# atomically move into place
61+
os.rename(tmpl_temp_name, tmpl_final_name)
62+
63+
if __name__ == '__main__':
64+
main()

bin/update/deploy-dev.py

+7
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,16 @@
77
import os
88
import sys
99

10+
from commander.deploy import task
11+
1012
# these files are symlinked as 'update.py' in the project root.
1113
ROOT = os.path.dirname(os.path.abspath(__file__))
1214
sys.path.append(ROOT)
1315
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
1416

1517
from deploy_dev_base import * # noqa
18+
19+
20+
@task
21+
def update_cron(ctx):
22+
generate_cron_file(ctx, 'bedrock-dev')

bin/update/deploy-prod.py

+7
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,16 @@
77
import os
88
import sys
99

10+
from commander.deploy import task
11+
1012
# these files are symlinked as 'update.py' in the project root.
1113
ROOT = os.path.dirname(os.path.abspath(__file__))
1214
sys.path.append(ROOT)
1315
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
1416

1517
from deploy_base import * # noqa
18+
19+
20+
@task
21+
def update_cron(ctx):
22+
generate_cron_file(ctx, 'bedrock-prod')

bin/update/deploy-stage.py

+7
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,16 @@
77
import os
88
import sys
99

10+
from commander.deploy import task
11+
1012
# these files are symlinked as 'update.py' in the project root.
1113
ROOT = os.path.dirname(os.path.abspath(__file__))
1214
sys.path.append(ROOT)
1315
sys.path.append(os.path.join(ROOT, 'bedrock', 'bin', 'update'))
1416

1517
from deploy_base import * # noqa
18+
19+
20+
@task
21+
def update_cron(ctx):
22+
generate_cron_file(ctx, 'bedrock-stage')

bin/update/deploy_base.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import commander_settings as settings
1616

1717

18+
PYTHON = getattr(settings, 'PYTHON_PATH', 'python2.6')
1819
NEW_RELIC_API_KEY = getattr(settings, 'NEW_RELIC_API_KEY', None)
1920
NEW_RELIC_APP_ID = getattr(settings, 'NEW_RELIC_APP_ID', None)
2021
NEW_RELIC_URL = 'https://rpm.newrelic.com/deployments.xml'
@@ -24,7 +25,7 @@
2425
def management_cmd(ctx, cmd):
2526
"""Run a Django management command correctly."""
2627
with ctx.lcd(settings.SRC_DIR):
27-
ctx.local('LANG=en_US.UTF-8 python2.6 manage.py ' + cmd)
28+
ctx.local('LANG=en_US.UTF-8 {0} manage.py {1}'.format(PYTHON, cmd))
2829

2930

3031
@task
@@ -139,6 +140,8 @@ def update(ctx):
139140

140141
@task
141142
def deploy(ctx):
143+
if 'update_cron' in commands:
144+
commands['update_cron']()
142145
commands['checkin_changes']()
143146
commands['deploy_app']()
144147
commands['ping_newrelic']()
@@ -187,3 +190,12 @@ def generate_desc(from_commit, to_commit, changelog):
187190
else:
188191
desc = get_random_desc()
189192
return desc
193+
194+
195+
def generate_cron_file(ctx, tmpl_name):
196+
with ctx.lcd(settings.SRC_DIR):
197+
ctx.local("{python} bin/gen-crons.py -p {python} -s {src_dir} -w {www_dir} "
198+
"-t {template}".format(python=PYTHON,
199+
src_dir=settings.SRC_DIR,
200+
www_dir=settings.WWW_DIR,
201+
template=tmpl_name))

bin/update/deploy_dev_base.py

+7-8
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@
1010

1111
@task
1212
def database(ctx):
13-
with ctx.lcd(settings.SRC_DIR):
14-
# only ever run this one on demo and dev.
15-
ctx.local("python2.6 manage.py bedrock_truncate_database --yes-i-am-sure")
16-
ctx.local("python2.6 manage.py syncdb --migrate --noinput")
17-
ctx.local("python2.6 manage.py rnasync")
18-
ctx.local("python2.6 manage.py update_security_advisories --force --quiet")
19-
ctx.local("python2.6 manage.py cron update_reps_ical")
20-
ctx.local("python2.6 manage.py cron update_tweets")
13+
# only ever run this one on demo and dev.
14+
management_cmd(ctx, 'bedrock_truncate_database --yes-i-am-sure')
15+
management_cmd(ctx, 'syncdb --migrate --noinput')
16+
management_cmd(ctx, 'rnasync')
17+
management_cmd(ctx, 'update_security_advisories --force --quiet')
18+
management_cmd(ctx, 'cron update_reps_ical')
19+
management_cmd(ctx, 'cron update_tweets')

etc/cron.d/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
bedrock-*
2+
!bedrock-*.tmpl

etc/cron.d/bedrock-dev

-10
This file was deleted.

etc/cron.d/bedrock-dev.tmpl

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
# {{ header }}
3+
#
4+
5+
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
6+
7+
0-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org.sh
8+
1-59/10 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-django.sh
9+
0 * * * * {{ django_cron }} update_feeds > /dev/null 2>&1
10+
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
11+
12+
*/15 * * * * root /data/bedrock-dev/src/update-www-dev.allizom.org-svn-locale.sh
13+
14+
# bug 1014586
15+
3 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
16+

etc/cron.d/bedrock-prod

-27
This file was deleted.

etc/cron.d/bedrock-prod.tmpl

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#
2+
# {{ header }}
3+
#
4+
5+
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
6+
7+
*/10 * * * * root /data/bedrock/src/update-www.mozilla.org.sh
8+
9+
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
10+
*/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-cron-svn-locale.sh &> /dev/null
11+
# outputs when the run completed to http://www.mozilla.org/media/locale_finished.txt
12+
13+
# disabled per bug 1042074
14+
#1-59/15 * * * * {{ user }} /data/bedrock/src/update-www.mozilla.org-product-details.sh
15+
16+
# bug 996144 & 1014586
17+
2 */6 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
18+
19+
# bug 1072999, 1109344
20+
*/30 * * * * {{ django_manage }} update_security_advisories --quiet > /dev/null 2>&1
21+
22+
# bug 1087533
23+
42 * * * * {{ django_cron }} update_reps_ical > /dev/null 2>&1
24+
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
1+
#
2+
# {{ header }}
3+
#
4+
15
MAILTO="webops-cron@mozilla.com,cron-bedrock@mozilla.com"
2-
BEDROCK=/data/bedrock-stage/src/www.allizom.org-django/bedrock
3-
BEDROCK_DEPLOY=/data/bedrock-stage/deploy
4-
# TODO: update rest of cron jobs to use the above env variables
56

6-
# webdev requests that prod auto-updates the PHP side, for similar to old
7-
0-59/10 * * * * root /data/bedrock-stage/src/update-www.allizom.org.sh
7+
*/10 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org.sh
88

9-
*/15 * * * * root /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
9+
*/15 * * * * {{ user }} /data/bedrock-stage/src/update-www.allizom.org-svn-locale.sh
1010
# outputs when the run completed to http://www.allizom.org/media/locale_finished.txt
1111

1212
# In testing for bug 753566 resolution...
1313
# not completely sure how I want to solve this yet
1414
#
1515
# run the crons from the admin node, but on the web nodes... does extra work, but works
16-
#0 * * * * * root echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
17-
*/5 * * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py rnasync > /dev/null 2>&1
16+
#0 * * * * * {{ user }} echo "cd /data/www/www.allizom.org-django/bedrock; python manage.py cron update_feeds | /usr/bin/issue-multi-command bedrock-stage
17+
*/5 * * * * {{ django_manage }} rnasync > /dev/null 2>&1
1818

1919
# bug 996144
20-
1 */2 * * * root cd /data/bedrock-stage/src/www.allizom.org-django/bedrock; python manage.py cron update_tweets > /dev/null 2>&1
20+
1 */2 * * * {{ django_cron }} update_tweets > /dev/null 2>&1
2121

2222
#
2323
# alternative- run the cron twice right on the admin node, passing --settings=settings/memcache-XXX.py... will this work?
@@ -26,9 +26,9 @@ BEDROCK_DEPLOY=/data/bedrock-stage/deploy
2626
#0 * * * * * root cd /data/www.allizom.org-django/bedrock; python manage.py --settings=settings/memcache-scl3.py cron update_feeds 2>&1 1> /dev/null | grep -v 'old-style Playdoh layout'"
2727

2828
# bug 1072999
29-
41 * * * * root cd $BEDROCK && python manage.py update_security_advisories --quiet
29+
41 * * * * {{ django_manage }} update_security_advisories --quiet
3030

3131
# bug 1087533
3232
# TODO: redirect output to null after initial deployment and verification
33-
42 * * * * root cd $BEDROCK && python manage.py cron update_reps_ical
33+
42 * * * * {{ django_cron }} update_reps_ical
3434

0 commit comments

Comments
 (0)