Merge branch 'couch_config'

corehq/apps/domainsync/management/commands/copy_case.py

@@ -5,6 +5,7 @@
 from casexml.apps.case.models import CommCareCase
 from corehq.apps.domainsync.management.commands.copy_utils import copy_postgres_data_for_docs
 from corehq.util.couch_helpers import OverrideDB
+from corehq.util.couchdb_management import CouchConfig
 from couchforms.models import XFormInstance
 
 
@@ -24,15 +25,15 @@ class Command(LabelCommand):
     def handle(self, *args, **options):
         if len(args) < 2:
             raise CommandError('Usage is copy_case, %s' % self.args)
-
-        sourcedb = Database(args[0])
+        source_couch = CouchConfig(args[0])
         case_id = args[1]
         doc_ids = [case_id]
 
         domain = args[2] if len(args) > 2 else None
+
         def _migrate_case(case_id):
             print 'getting case %s' % case_id
-            case = CommCareCase.wrap(sourcedb.get(case_id))
+            case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id))
             original_domain = case.domain
             if domain is not None:
                 case.domain = domain
@@ -47,7 +48,7 @@ def _migrate_case(case_id):
 
         # hack, set the domain back to make sure we get the reverse indices correctly
         case.domain = orig_domain
-        with OverrideDB(CommCareCase, sourcedb):
+        with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)):
             child_indices = get_reverse_indices(case)
         print 'copying %s child cases' % len(child_indices)
         for index in child_indices:
@@ -61,7 +62,7 @@ def form_wrapper(row):
             doc.pop('_attachments', None)
             return XFormInstance.wrap(doc)
 
-        xforms = sourcedb.all_docs(
+        xforms = source_couch.get_db_for_class(XFormInstance).all_docs(
             keys=case.xform_ids,
             include_docs=True,
             wrapper=form_wrapper,

corehq/apps/domainsync/management/commands/copy_domain.py

@@ -8,6 +8,7 @@
 from restkit import RequestError
 from corehq.apps.domain.models import Domain
 from corehq.apps.domainsync.management.commands.copy_utils import copy_postgres_data_for_docs
+from corehq.util.couchdb_management import CouchConfig
 from corehq.util.dates import iso_string_to_date
 from dimagi.utils.couch.database import get_db, iter_docs
 from corehq.apps.domainsync.config import DocumentTransform, save
@@ -91,11 +92,23 @@ class Command(BaseCommand):
                          "copy. This must be false if running in a supervised process")
     )
 
+    def iter_source_dbs(self):
+        for sourcedb_name, sourcedb in self.source_couch.all_dbs_by_slug.items():
+            if sourcedb_name not in self.exclude_dbs:
+                print "In {} db".format(sourcedb_name or "the main")
+                yield sourcedb_name, sourcedb
+
     def handle(self, *args, **options):
         if len(args) not in [2, 3]:
             raise CommandError('Usage is copy_domain %s' % self.args)
-
-        sourcedb = Database(args[0])
+        self.exclude_dbs = (
+            # these have data we don't want to copy
+            'receiverwrapper', 'couchlog', 'auditcare', 'fluff-bihar', 'fluff-opm',
+            'fluff-mc', 'fluff-cvsu', 'mvp-indicators', 'm4change',
+            # todo: missing domain/docs, but probably want to add back
+            'meta',
+        )
+        self.source_couch = source_couch = CouchConfig(args[0])
         domain = args[1].strip()
         simulate = options['simulate']
         exclude_attachments = options['exclude_attachments']
@@ -104,7 +117,8 @@ def handle(self, *args, **options):
         since = json_format_date(iso_string_to_date(options['since'])) if options['since'] else None
 
         if options['list_types']:
-            self.list_types(sourcedb, domain, since)
+            for sourcedb_name, sourcedb in self.iter_source_dbs():
+                self.list_types(sourcedb, domain, since)
             sys.exit(0)
 
         if simulate:
@@ -121,14 +135,15 @@ def handle(self, *args, **options):
             domain_doc = None
 
         if domain_doc is None:
-            self.copy_domain(sourcedb, domain)
+            self.copy_domain(source_couch, domain)
 
         if options['doc_types']:
             doc_types = options['doc_types'].split(',')
-            for type in doc_types:
-                startkey = [x for x in [domain, type, since] if x is not None]
-                endkey = [x for x in [domain, type, {}] if x is not None]
-                self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=type, since=since,
+            for doc_type in doc_types:
+                sourcedb = source_couch.get_db_for_doc_type(doc_type)
+                startkey = [x for x in [domain, doc_type, since] if x is not None]
+                endkey = [x for x in [domain, doc_type, {}] if x is not None]
+                self.copy_docs(sourcedb, domain, simulate, startkey, endkey, doc_type=doc_type, since=since,
                                postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments)
         elif options['id_file']:
             path = options['id_file']
@@ -143,14 +158,16 @@ def handle(self, *args, **options):
                 print "Path '%s' does not contain any document ID's" % path
                 sys.exit(1)
 
-            self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'],
-                           exclude_attachments=exclude_attachments)
+            for sourcedb_name, sourcedb in self.iter_source_dbs():
+                self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'],
+                               exclude_attachments=exclude_attachments)
         else:
             startkey = [domain]
             endkey = [domain, {}]
             exclude_types = DEFAULT_EXCLUDE_TYPES + options['doc_types_exclude'].split(',')
-            self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types,
-                           postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments)
+            for sourcedb_name, sourcedb in self.iter_source_dbs():
+                self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types,
+                               postgres_db=options['postgres_db'], exclude_attachments=exclude_attachments)
 
     def list_types(self, sourcedb, domain, since):
         doc_types = sourcedb.view("domain/docs", startkey=[domain],
@@ -208,8 +225,9 @@ def copy_docs(self, sourcedb, domain, simulate, startkey=None, endkey=None, doc_
         if postgres_db:
             copy_postgres_data_for_docs(postgres_db, doc_ids=doc_ids, simulate=simulate)
 
-    def copy_domain(self, sourcedb, domain):
+    def copy_domain(self, source_couch, domain):
         print "Copying domain doc"
+        sourcedb = source_couch.get_db_for_class(Domain)
         result = sourcedb.view(
             "domain/domains",
             key=domain,

corehq/util/couch.py

@@ -62,31 +62,44 @@ def get_document_or_404(cls, domain, doc_id, additional_doc_types=None):
 
 
 @memoized
-def get_document_class_by_name(name):
+def get_classes_by_doc_type():
+    queue = [Document]
+    classes_by_doc_type = {}
+    while queue:
+        klass = queue.pop()
+        try:
+            klass._meta.app_label
+        except AttributeError:
+            # exclude abstract base classes (which don't have an app_label)
+            pass
+        else:
+            # a base class (e.g. CommCareCase) wins over a subclass (e.g. BiharCase)
+            if klass._doc_type not in classes_by_doc_type:
+                classes_by_doc_type[klass._doc_type] = klass
+        queue.extend(klass.__subclasses__())
+    return classes_by_doc_type
+
+
+def get_document_class_by_doc_type(doc_type):
     """
-    Given the name of a document class, get the class itself.
+    Given the doc_type of a document class, get the class itself.
 
     Raises a DocumentClassNotFound if not found
     """
-    def _all_subclasses(cls):
-        for subclass in cls.__subclasses__():
-            yield subclass
-            for subsubclass in _all_subclasses(subclass):
-                yield subsubclass
-
-    for subclass in _all_subclasses(Document):
-        if subclass.__name__ == name:
-            return subclass
 
-    raise DocumentClassNotFound(u'No Document class with name "{}" could be found.'.format(name))
+    try:
+        return get_classes_by_doc_type()[doc_type]
+    except KeyError:
+        raise DocumentClassNotFound(
+            u'No Document class with name "{}" could be found.'.format(doc_type))
 
 
 def get_db_by_doc_type(doc_type):
     """
     Lookup a database by document type. Returns None if the database is not found.
     """
     try:
-        return get_document_class_by_name(doc_type).get_db()
+        return get_document_class_by_doc_type(doc_type).get_db()
     except DocumentClassNotFound:
         return None
 

corehq/util/couchdb_management.py

@@ -0,0 +1,63 @@
+from couchdbkit.client import Database
+from corehq.util.couch import get_document_class_by_doc_type
+from dimagi.utils.decorators.memoized import memoized
+from django.conf import settings
+
+
+class CouchConfig(object):
+    """
+    Interface for accessing the couch-related settings
+
+    You can use db_uri pass in a different couch instance
+    from the one specified by COUCH_DATABASE
+
+    """
+    def __init__(self, db_uri=None):
+        if db_uri:
+            self._settings_helper = (
+                settings.COUCH_SETTINGS_HELPER._replace(couch_database_url=db_uri))
+        else:
+            self._settings_helper = settings.COUCH_SETTINGS_HELPER
+
+    @property
+    def db_uri(self):
+        return self._settings_helper.couch_database_url
+
+    @property
+    @memoized
+    def all_db_uris_by_slug(self):
+        dbs = self._settings_helper.get_extra_couchdbs()
+        dbs[None] = self.db_uri
+        return dbs
+
+    @property
+    @memoized
+    def all_dbs_by_slug(self):
+        return {slug: Database(db_uri)
+                for slug, db_uri in self.all_db_uris_by_slug.items()}
+
+    def get_db(self, postfix):
+        """
+        Get the couch database by slug
+        """
+        return Database(self.all_db_uris_by_slug[postfix], create=True)
+
+    @property
+    @memoized
+    def app_label_to_db_uri(self):
+        return dict(self._settings_helper.make_couchdb_tuples())
+
+    def get_db_uri_for_class(self, klass):
+        return self.app_label_to_db_uri[getattr(klass._meta, "app_label")]
+
+    def get_db_uri_for_doc_type(self, doc_type):
+        return self.get_db_uri_for_class(get_document_class_by_doc_type(doc_type))
+
+    def get_db_for_class(self, klass):
+        return Database(self.get_db_uri_for_class(klass))
+
+    def get_db_for_doc_type(self, doc_type):
+        return Database(self.get_db_uri_for_doc_type(doc_type))
+
+
+couch_config = CouchConfig()

corehq/util/tests/__init__.py

@@ -1,16 +1,17 @@
-from corehq.util.spreadsheets.excel import IteratorJSONReader
+from test_cache_util import *
 from test_couch import *
+from test_couchdb_management import *
+from test_jsonobject import *
 from test_log import *
-from test_toggle import *
+from test_override_db import *
 from test_quickcache import *
-from test_timezone_conversions import *
 from test_soft_assert import *
-from test_cache_util import *
-from test_override_db import *
 from test_spreadsheets import *
-from test_jsonobject import *
+from test_timezone_conversions import *
+from test_toggle import *
 from test_xml import *
 
+from corehq.util.spreadsheets.excel import IteratorJSONReader
 from corehq.util.dates import iso_string_to_datetime, iso_string_to_date
 
 __test__ = {

corehq/util/tests/test_couch.py

@@ -9,7 +9,7 @@
 from corehq.util.exceptions import DocumentClassNotFound
 
 from ..couch import (get_document_or_404, IterDB, iter_update, IterUpdateError,
-        DocUpdate, get_document_class_by_name)
+        DocUpdate, get_document_class_by_doc_type)
 
 
 class MockDb(object):
@@ -253,10 +253,10 @@ def test_a_few_important_ones(self):
             ('FixtureDataType', FixtureDataType),
         ]
         for model_name, model_class in test_cases:
-            self.assertEqual(model_class, get_document_class_by_name(model_name))
+            self.assertEqual(model_class, get_document_class_by_doc_type(model_name))
 
     def test_missing(self):
         test_cases = [None, 'FooDocument']
         for bad_model in test_cases:
             with self.assertRaises(DocumentClassNotFound):
-                get_document_class_by_name(bad_model)
+                get_document_class_by_doc_type(bad_model)

corehq/util/tests/test_couchdb_management.py

@@ -0,0 +1,36 @@
+from django.conf import settings
+from django.test import SimpleTestCase
+from corehq.util.couchdb_management import CouchConfig, couch_config
+
+
+class CouchConfigTest(SimpleTestCase):
+    remote_db_uri = 'https://mycouch.com/cchq'
+
+    def test_default_db_uri(self):
+        config = CouchConfig()
+        self.assertEqual(config.db_uri, settings.COUCH_DATABASE)
+
+    def test_default_couch_config_db_uri(self):
+        self.assertEqual(couch_config.db_uri, settings.COUCH_DATABASE)
+
+    def test_remote_db_uri(self):
+        config = CouchConfig(db_uri=self.remote_db_uri)
+        self.assertEqual(config.db_uri, self.remote_db_uri)
+
+    def test_all_db_uris_by_slug(self):
+        config = CouchConfig(db_uri=self.remote_db_uri)
+        self.assertDictContainsSubset(
+            {
+                None: self.remote_db_uri,
+                'users': '{}__users'.format(self.remote_db_uri),
+                'fixtures': '{}__fixtures'.format(self.remote_db_uri),
+                'meta': '{}__meta'.format(self.remote_db_uri),
+            },
+            config.all_db_uris_by_slug
+        )
+
+    def test_get_db_for_doc_type(self):
+        config = CouchConfig(db_uri=self.remote_db_uri)
+        self.assertEqual(config.get_db_for_doc_type('CommCareCase').uri, self.remote_db_uri)
+        self.assertEqual(config.get_db_for_doc_type('CommCareUser').uri,
+                         '{}__users'.format(self.remote_db_uri))

settings.py

@@ -1035,8 +1035,7 @@
 ####### Couch Forms & Couch DB Kit Settings #######
 from settingshelper import (
     get_dynamic_db_settings,
-    make_couchdb_tuples,
-    get_extra_couchdbs,
+    CouchSettingsHelper,
     SharedDriveConfiguration
 )
 
@@ -1157,12 +1156,11 @@
 
 COUCHDB_APPS += LOCAL_COUCHDB_APPS
 
-COUCHDB_DATABASES = make_couchdb_tuples(COUCHDB_APPS, COUCH_DATABASE)
-EXTRA_COUCHDB_DATABASES = get_extra_couchdbs(COUCHDB_APPS, COUCH_DATABASE, (
-    NEW_USERS_GROUPS_DB,
-    NEW_FIXTURES_DB,
-    NEW_DOMAINS_DB,
-))
+COUCH_SETTINGS_HELPER = CouchSettingsHelper(COUCH_DATABASE, COUCHDB_APPS, [
+    NEW_USERS_GROUPS_DB, NEW_FIXTURES_DB, NEW_DOMAINS_DB,
+])
+COUCHDB_DATABASES = COUCH_SETTINGS_HELPER.make_couchdb_tuples()
+EXTRA_COUCHDB_DATABASES = COUCH_SETTINGS_HELPER.get_extra_couchdbs()
 
 INSTALLED_APPS += LOCAL_APPS