Merge pull request #3324 from robhudson/docker-celery

Update docker to use celery via redis

Dockerfile

@@ -42,5 +42,10 @@ ENV SOLITUDE_URL http://solitude:2602
 ENV STYLUS_BIN /srv/zamboni-node/node_modules/stylus/bin/stylus
 ENV UGLIFY_BIN /srv/zamboni-node/node_modules/uglify-js/bin/uglifyjs
 ENV ZAMBONI_DATABASE mysql://root:@mysql:3306/zamboni
+ENV CELERY_BROKER_URL redis://redis:6379/1
+ENV CELERY_RESULT_BACKEND redis://redis:6379/2
+# Required to run celery as root using pickle serialization.
+# TODO: Remove when we move to json serialization.
+ENV C_FORCE_ROOT 1
 
 EXPOSE 2600

mkt/settings.py

@@ -103,8 +103,7 @@ def path(*args):
 DEFAULT_FROM_EMAIL = 'Firefox Marketplace <nobody@mozilla.org>'
 
 # The host currently running the site, used for browserid and other lookups.
-DOMAIN = urlparse(os.environ.get('MARKETPLACE_URL',
-                                 'http://localhost')).netloc
+DOMAIN = urlparse(os.getenv('MARKETPLACE_URL', 'http://localhost')).netloc
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
@@ -403,21 +402,21 @@ def path(*args):
 
 ###########################################
 # Celery
-BROKER_URL = 'amqp://zamboni:zamboni@{0}:5672/zamboni'.format(
-             os.environ.get('RABBIT_HOST', 'localhost'))
+BROKER_URL = os.getenv('CELERY_BROKER_URL',
+                       'amqp://zamboni:zamboni@localhost:5672/zamboni')
 BROKER_CONNECTION_TIMEOUT = 0.1
 
 CEF_PRODUCT = 'mkt'
 
-CELERY_ALWAYS_EAGER = True
+CELERY_ALWAYS_EAGER = False
 
 # Testing responsiveness without rate limits.
 CELERY_DISABLE_RATE_LIMITS = True
 
 CELERY_IGNORE_RESULT = True
 CELERY_IMPORTS = ('lib.video.tasks', 'lib.metrics',
                   'lib.es.management.commands.reindex')
-CELERY_RESULT_BACKEND = 'amqp'
+CELERY_RESULT_BACKEND = os.getenv('CELERY_RESULT_BACKEND', 'amqp')
 CELERY_ACCEPT_CONTENT = ['pickle']
 CELERY_TASK_SERIALIZER = 'pickle'
 CELERY_RESULT_SERIALIZER = 'pickle'
@@ -592,8 +591,8 @@ def langs(languages):
     build_id = ""
 
 # Path to cleancss (our CSS minifier).
-CLEANCSS_BIN = os.environ.get('CLEANCSS_BIN',
-                              path('node_modules/clean-css/bin/cleancss'))
+CLEANCSS_BIN = os.getenv('CLEANCSS_BIN',
+                         path('node_modules/clean-css/bin/cleancss'))
 
 # Name of our frontend repositories on GitHub.
 COMMONPLACE_REPOS = ['commbadge', 'fireplace', 'marketplace-stats',
@@ -684,7 +683,7 @@ def langs(languages):
 # replicas to zero.
 ES_DEFAULT_NUM_REPLICAS = 0
 ES_DEFAULT_NUM_SHARDS = 5
-ES_HOSTS = [os.environ.get('ES_HOST', '127.0.0.1:9200')]
+ES_HOSTS = [os.getenv('ES_HOST', '127.0.0.1:9200')]
 ES_INDEXES = {
     'webapp': 'apps',
     'extension': 'extensions',
@@ -754,7 +753,7 @@ def langs(languages):
         'e9c1237312e141a0294715c2fac6855ff06ab4f5103b9cfa394cc83ea6653f26',
 }
 
-FXA_CLIENT_ID = os.environ.get('FXA_CLIENT_ID', '6b00a7db54f9efee')
+FXA_CLIENT_ID = os.getenv('FXA_CLIENT_ID', '6b00a7db54f9efee')
 FXA_CLIENT_SECRET = FXA_SECRETS[FXA_CLIENT_ID]
 
 # Development environments don't use HTTPS, so we disable oauthlib's URL check.
@@ -920,7 +919,7 @@ def JINJA_CONFIG():
 MINIFY_MOZMARKET = True
 
 # Monolith settings.
-MONOLITH_SERVER = os.environ.get('MONOLITH_URL', 'http://localhost:9200')
+MONOLITH_SERVER = os.getenv('MONOLITH_URL', 'http://localhost:9200')
 MONOLITH_INDEX = 'time_*'
 MONOLITH_MAX_DATE_RANGE = 365
 
@@ -1064,7 +1063,7 @@ def JINJA_CONFIG():
 SIGNED_APPS_OMIT_PER_FILE_SIGS = True
 
 # This is the signing REST server for signing receipts.
-SIGNING_SERVER = os.environ.get('SIGNING_SERVER', '')
+SIGNING_SERVER = os.getenv('SIGNING_SERVER', '')
 
 # Turn on/off the use of the signing server and all the related things. This
 # is a temporary flag that we will remove.
@@ -1081,7 +1080,7 @@ def JINJA_CONFIG():
 
 # The configuration for the client that speaks to solitude.
 # A tuple of the solitude hosts.
-SOLITUDE_HOSTS = (os.environ.get('SOLITUDE_URL', 'http://localhost:2602'),)
+SOLITUDE_HOSTS = (os.getenv('SOLITUDE_URL', 'http://localhost:2602'),)
 
 # The oAuth key and secret that solitude needs.
 SOLITUDE_KEY = 'marketplace'
@@ -1119,8 +1118,7 @@ def JINJA_CONFIG():
 STATSD_CLIENT = 'django_statsd.clients.normal'
 
 # Path to stylus (to compile .styl files).
-STYLUS_BIN = os.environ.get('STYLUS_BIN',
-                            path('node_modules/stylus/bin/stylus'))
+STYLUS_BIN = os.getenv('STYLUS_BIN', path('node_modules/stylus/bin/stylus'))
 SYSLOG_TAG = "http_app_addons"
 SYSLOG_TAG2 = "http_app_addons2"
 
@@ -1140,8 +1138,8 @@ def JINJA_CONFIG():
 TOWER_ADD_HEADERS = True
 
 # Path to uglifyjs (our JS minifier).
-UGLIFY_BIN = os.environ.get('UGLIFY_BIN',
-                            path('node_modules/uglify-js/bin/uglifyjs'))
+UGLIFY_BIN = os.getenv('UGLIFY_BIN',
+                       path('node_modules/uglify-js/bin/uglifyjs'))
 
 # Feature flags
 UNLINK_SITE_STATS = True

scripts/docker/supervisor.conf

@@ -18,3 +18,12 @@ port=9001
 
 [rpcinterface:supervisor]
 supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[program:celeryd]
+command=python manage.py celery worker --autoreload --broker=redis://redis:6379/1 -Q priority,devhub,images,limited,celery --loglevel DEBUG
+directory=/srv/zamboni
+stopasgroup=true
+autostart=true
+redirect_stderr=true
+stdout_logfile=logs/celery.log
+stdout_logfile_maxbytes=1MB