Revert 72fe88e and f76c11a since it's causing issues on dev/stage (bu…

…g 1159220)

mkt/receipts/tests/test_verify.py

@@ -4,6 +4,7 @@
 import uuid
 from urllib import urlencode
 
+from django.db import connection
 from django.conf import settings
 from django.test.client import RequestFactory
 
@@ -63,20 +64,13 @@ def sample_inapp_receipt(self, contribution):
 @mock.patch.object(settings, 'WEBAPPS_RECEIPT_URL', '/verifyme/')
 class TestVerify(ReceiptTest):
 
-    def test_setup_db(self):
-        verifier = verify.Verify('', RequestFactory().get('/verifyme/').META)
-        assert not verifier.connection
-        verifier.setup_db()
-        assert verifier.connection
-        assert verifier.connection.cursor()
-        assert (verifier.connection.get_connection_params()['db']
-                .startswith('test_'))
-
     def verify_signed_receipt(self, signed_receipt, check_purchase=True):
+        # Ensure that the verify code is using the test database cursor.
         verifier = verify.Verify(
             signed_receipt,
             RequestFactory().get('/verifyme/').META
         )
+        verifier.cursor = connection.cursor()
 
         if check_purchase:
             return verifier.check_full()
@@ -444,16 +438,8 @@ def test_good(self):
 
 class TestServices(mkt.site.tests.TestCase):
 
-    def test_settings(self):
-        # services.verify is not a real django app, it imports django settings
-        # manually, so we can't simply use self.settings() context manager.
-        # See https://bugzilla.mozilla.org/show_bug.cgi?id=1159746
-        with mock.patch('services.verify.settings') as settings:
-            settings.SIGNING_SERVER_ACTIVE = True
-            eq_(verify.status_check({})[0], 200)
-
-        with mock.patch('services.verify.settings') as settings:
-            settings.SIGNING_SERVER_ACTIVE = False
+    def test_wrong_settings(self):
+        with self.settings(SIGNING_SERVER_ACTIVE=''):
             eq_(verify.status_check({})[0], 500)
 
     def test_options_request_for_cors(self):

mkt/settings.py

@@ -1028,6 +1028,11 @@ def JINJA_CONFIG():
 
 SENTRY_DSN = None
 
+# A database to be used by the services scripts, which does not use Django.
+# The settings can be copied from DATABASES, but since its not a full Django
+# database connection, only some values are supported.
+SERVICES_DATABASE = DATABASES['default']
+
 SHORTER_LANGUAGES = {'en': 'en-US', 'ga': 'ga-IE', 'pt': 'pt-PT',
                      'sv': 'sv-SE', 'zh': 'zh-CN'}
 

requirements/prod.txt

@@ -103,6 +103,7 @@ signing_clients==0.1.7
 six==1.9.0
 # slumber is required by curling
 slumber==0.6.2
+SQLAlchemy==0.7.5
 statsd==2.0.3
 suds==0.4
 tower==0.4.1

services/utils.py

@@ -14,6 +14,9 @@
 
 import sys  # noqa
 
+import MySQLdb as mysql  # noqa
+import sqlalchemy.pool as pool  # noqa
+
 from django.utils import importlib  # noqa
 settings = importlib.import_module(settingmodule)
 
@@ -24,6 +27,15 @@
 from lib.log_settings_base import formatters, handlers  # noqa
 
 
+def getconn():
+    db = settings.SERVICES_DATABASE
+    return mysql.connect(host=db['HOST'], user=db['USER'],
+                         passwd=db['PASSWORD'], db=db['NAME'])
+
+
+mypool = pool.QueuePool(getconn, max_overflow=10, pool_size=5, recycle=300)
+
+
 def log_configure():
     """You have to call this to explicity configure logging."""
     cfg = {

services/verify.py

@@ -18,7 +18,8 @@
 from services.utils import settings
 
 from utils import (CONTRIB_CHARGEBACK, CONTRIB_NO_CHARGE, CONTRIB_PURCHASE,
-                   CONTRIB_REFUND, log_configure, log_exception, log_info)
+                   CONTRIB_REFUND, log_configure, log_exception, log_info,
+                   mypool)
 
 # Go configure the log.
 log_configure()
@@ -55,7 +56,9 @@ class Verify:
     def __init__(self, receipt, environ):
         self.receipt = receipt
         self.environ = environ
-        self.connection = None
+
+        # This is so the unit tests can override the connection.
+        self.conn, self.cursor = None, None
 
     def check_full(self):
         """
@@ -219,8 +222,9 @@ def setup_db(self):
         All database calls are done at a low level and avoid the
         Django ORM.
         """
-        from django.db import connections
-        self.connection = connections['default']
+        if not self.cursor:
+            self.conn = mypool.connect()
+            self.cursor = self.conn.cursor()
 
     def check_purchase(self):
         """
@@ -240,10 +244,11 @@ def check_purchase_inapp(self):
         sql = """SELECT i.guid, c.type FROM stats_contributions c
                  JOIN inapp_products i ON i.id=c.inapp_product_id
                  WHERE c.id = %(contribution_id)s LIMIT 1;"""
-        with self.connection.cursor() as cursor:
-            cursor.execute(
-                sql, {'contribution_id': self.get_contribution_id()})
-            result = cursor.fetchone()
+        self.cursor.execute(
+            sql,
+            {'contribution_id': self.get_contribution_id()}
+        )
+        result = self.cursor.fetchone()
         if not result:
             log_info('Invalid in-app receipt, no purchase')
             raise InvalidReceipt('NO_PURCHASE')
@@ -265,10 +270,9 @@ def check_purchase_app(self):
         sql = """SELECT type FROM addon_purchase
                  WHERE addon_id = %(app_id)s
                  AND uuid = %(uuid)s LIMIT 1;"""
-        with self.connection.cursor() as cursor:
-            cursor.execute(sql, {'app_id': self.get_app_id(),
-                                 'uuid': self.get_user()})
-            result = cursor.fetchone()
+        self.cursor.execute(sql, {'app_id': self.get_app_id(),
+                                  'uuid': self.get_user()})
+        result = self.cursor.fetchone()
         if not result:
             log_info('Invalid app receipt, no purchase')
             raise InvalidReceipt('NO_PURCHASE')
@@ -395,10 +399,9 @@ def status_check(environ):
         return 500, 'SIGNING_SERVER_ACTIVE is not set'
 
     try:
-        from django.db import connections
-        with connections['default'].cursor() as cursor:
-            cursor.execute(
-                'SELECT id FROM users_install ORDER BY id DESC LIMIT 1')
+        conn = mypool.connect()
+        cursor = conn.cursor()
+        cursor.execute('SELECT id FROM users_install ORDER BY id DESC LIMIT 1')
     except Exception, err:
         return 500, str(err)
 

sites/altdev/settings_base.py

@@ -38,6 +38,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {

sites/dev/settings_base.py

@@ -39,6 +39,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {

sites/identitystage/settings_base.py

@@ -35,6 +35,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {

sites/paymentsalt/settings_base.py

@@ -38,6 +38,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {

sites/prod/settings_base.py

@@ -41,6 +41,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {

sites/stage/settings_base.py

@@ -38,6 +38,8 @@
 DATABASES['slave']['ATOMIC_REQUESTS'] = True
 DATABASES['slave']['CONN_MAX_AGE'] = 5 * 60  # 5m for persistent connections.
 
+SERVICES_DATABASE = dj_database_url.parse(private.SERVICES_DATABASE_URL)
+
 SLAVE_DATABASES = ['slave']
 
 CACHES = {