[couch to sql] UserRole & Permissions: Part I - sql models + populate command

corehq/apps/cleanup/management/commands/populate_sql_model_from_couch_model.py

@@ -53,6 +53,18 @@ def diff_couch_and_sql(cls, couch, sql):
         """
         raise NotImplementedError()
 
+    @classmethod
+    def get_filtered_diffs(cls, couch, sql):
+        diffs = cls.diff_couch_and_sql(couch, sql)
+        if isinstance(diffs, list):
+            diffs = list(filter(None, diffs))
+        return diffs
+
+    @classmethod
+    def get_diff_as_string(cls, couch, sql):
+        diffs = cls.get_filtered_diffs(couch, sql)
+        return "\n".join(diffs) if diffs else None
+
     @classmethod
     def diff_attr(cls, name, doc, obj, wrap_couch=None, wrap_sql=None, name_prefix=None):
         """
@@ -201,14 +213,6 @@ def handle(self, **options):
         if not skip_verify:
             logger.info(f"Found {self.diff_count} differences")
 
-    @classmethod
-    def get_diff_as_string(cls, couch, sql):
-        diff = cls.diff_couch_and_sql(couch, sql)
-        if isinstance(diff, list):
-            diffs = list(filter(None, diff))
-            diff = "\n".join(diffs) if diffs else None
-        return diff
-
     def _verify_doc(self, doc, exit=True):
         try:
             couch_id_name = getattr(self.sql_class(), '_migration_couch_id_name', 'couch_id')

corehq/apps/domain/deletion.py

@@ -381,6 +381,9 @@ def _delete_demo_user_restores(domain_name):
     ModelDeletion('users', 'Invitation', 'domain'),
     ModelDeletion('users', 'DomainPermissionsMirror', 'source'),
     ModelDeletion('users', 'UserReportingMetadataStaging', 'domain'),
+    ModelDeletion('users', 'SQLUserRole', 'domain', [
+        'RolePermission', 'RoleAssignableBy', 'SQLPermission'
+    ]),
     ModelDeletion('user_importer', 'UserUploadRecord', 'domain'),
     ModelDeletion('zapier', 'ZapierSubscription', 'domain'),
     ModelDeletion('dhis2', 'SQLDataValueMap', 'dataset_map__domain'),

corehq/apps/domain/tests/test_delete_domain.py

@@ -65,7 +65,6 @@
     CaseRuleSubmission,
     DomainCaseRuleRun,
 )
-from corehq.apps.domain.dbaccessors import get_docs_in_domain_by_class
 from corehq.apps.domain.models import Domain, TransferDomainRequest
 from corehq.apps.export.models.new import DataFile, EmailExportWhenDoneRequest
 from corehq.apps.ivr.models import Call
@@ -100,7 +99,10 @@
 from corehq.apps.smsforms.models import SQLXFormsSession
 from corehq.apps.translations.models import SMSTranslations, TransifexBlacklist
 from corehq.apps.userreports.models import AsyncIndicator
-from corehq.apps.users.models import DomainRequest, Invitation
+from corehq.apps.users.models import (
+    DomainRequest, Invitation, PermissionInfo, Permissions,
+    SQLUserRole, RolePermission, RoleAssignableBy
+)
 from corehq.apps.zapier.consts import EventTypes
 from corehq.apps.zapier.models import ZapierSubscription
 from corehq.blobs import CODES, NotFound, get_blob_db
@@ -875,6 +877,32 @@ def test_users_delete(self):
         self._assert_users_counts(self.domain.name, 0)
         self._assert_users_counts(self.domain2.name, 1)
 
+    def _assert_role_counts(self, domain_name, roles, permissions, assignments):
+        self.assertEqual(SQLUserRole.objects.filter(domain=domain_name).count(), roles)
+        self.assertEqual(RolePermission.objects.filter(role__domain=domain_name).count(), permissions)
+        self.assertEqual(RoleAssignableBy.objects.filter(role__domain=domain_name).count(), assignments)
+
+    def test_roles_delete(self):
+        for domain_name in [self.domain.name, self.domain2.name]:
+            role1 = SQLUserRole.objects.create(
+                domain=domain_name,
+                name="role1"
+            )
+            role = SQLUserRole.objects.create(
+                domain=domain_name,
+                name="role2"
+            )
+            role.set_permissions([
+                PermissionInfo(Permissions.view_reports.name, allow=PermissionInfo.ALLOW_ALL)
+            ])
+            role.set_assignable_by([role1.id])
+            self._assert_role_counts(domain_name, 2, 1, 1)
+
+        self.domain.delete()
+
+        self._assert_role_counts(self.domain.name, 0, 0, 0)
+        self._assert_role_counts(self.domain2.name, 2, 1, 1)
+
     def _assert_zapier_counts(self, domain_name, count):
         self._assert_queryset_count([
             ZapierSubscription.objects.filter(domain=domain_name),

corehq/apps/dump_reload/sql/dump.py

@@ -126,6 +126,7 @@
     FilteredModelIteratorBuilder('linked_domain.DomainLink', SimpleFilter('linked_domain')),
     FilteredModelIteratorBuilder('linked_domain.DomainLinkHistory', SimpleFilter('link__linked_domain')),
     FilteredModelIteratorBuilder('users.DomainPermissionsMirror', SimpleFilter('source')),
+    FilteredModelIteratorBuilder('users.SQLUserRole', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('locations.LocationFixtureConfiguration', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('commtrack.SQLCommtrackConfig', SimpleFilter('domain')),
     FilteredModelIteratorBuilder('commtrack.SQLActionConfig', SimpleFilter('commtrack_config__domain')),

corehq/apps/linked_domain/tests/test_update_roles.py

@@ -1,7 +1,13 @@
+import uuid
+
+from django.test import TestCase
 from mock import patch
 
+from corehq.apps.domain.shortcuts import create_domain
+from corehq.apps.linked_domain.models import DomainLink
 from corehq.apps.linked_domain.tests.test_linked_apps import BaseLinkedAppsTest
 from corehq.apps.linked_domain.updates import update_user_roles
+from corehq.apps.linked_domain.util import _clean_json
 from corehq.apps.userreports.util import get_ucr_class_name
 from corehq.apps.users.models import Permissions, UserRole
 
@@ -10,8 +16,6 @@ class TestUpdateRoles(BaseLinkedAppsTest):
     @classmethod
     def setUpClass(cls):
         super(TestUpdateRoles, cls).setUpClass()
-        cls.linked_app.save()
-
         cls.role = UserRole(
             domain=cls.domain,
             name='test',
@@ -104,3 +108,93 @@ def test_match_ids(self):
         self.assertIsNotNone(roles.get('other_test'))
         self.assertTrue(roles['other_test'].permissions.edit_web_users)
         self.assertEqual(roles['other_test'].upstream_id, self.other_role.get_id)
+
+
+class TestUpdateRolesRemote(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        super(TestUpdateRolesRemote, cls).setUpClass()
+        cls.domain_obj = create_domain('domain')
+        cls.domain = cls.domain_obj.name
+
+        cls.linked_domain_obj = create_domain('domain-2')
+        cls.linked_domain = cls.linked_domain_obj.name
+
+        cls.domain_link = DomainLink.link_domains(cls.linked_domain, cls.domain)
+        cls.domain_link.remote_base_url = "http://other.org"
+        cls.domain_link.save()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.domain_link.delete()
+        cls.domain_obj.delete()
+        cls.linked_domain_obj.delete()
+        super(TestUpdateRolesRemote, cls).tearDownClass()
+
+    def setUp(self):
+        self.upstream_role1_id = uuid.uuid4().hex
+        self.role1 = UserRole(
+            domain=self.linked_domain,
+            name='test',
+            permissions=Permissions(
+                edit_data=True,
+                edit_reports=True,
+                view_report_list=[
+                    'corehq.reports.DynamicReportmaster_report_id'
+                ]
+            ),
+            is_non_admin_editable=True,
+            upstream_id=self.upstream_role1_id
+        )
+        self.role1.save()
+
+        self.other_role = UserRole(
+            domain=self.linked_domain,
+            name='other_test',
+            permissions=Permissions(
+                edit_web_users=True,
+                view_locations=True,
+            ),
+            assignable_by=[self.role1.get_id],
+        )
+        self.other_role.save()
+
+    def tearDown(self):
+        for role in UserRole.by_domain(self.linked_domain):
+            role.delete()
+        super(TestUpdateRolesRemote, self).tearDown()
+
+    @patch('corehq.apps.linked_domain.updates.remote_get_user_roles')
+    def test_update_remote(self, remote_get_user_roles):
+        remote_permissions = Permissions(
+            edit_data=False,
+            edit_reports=True,
+            view_report_list=['corehq.reports.static_report']
+        )
+        # sync with existing local role
+        remote_role1 = UserRole(
+            _id=self.upstream_role1_id,
+            name='test',
+            permissions=remote_permissions,
+            is_non_admin_editable=False
+        )
+        # create new role
+        remote_role_other = UserRole(
+            _id=uuid.uuid4().hex,
+            name="another",
+            permissions=Permissions(),
+            assignable_by=[self.upstream_role1_id]
+        )
+        remote_get_user_roles.return_value = [
+            _clean_json(role.to_json()) for role in [remote_role1, remote_role_other]
+        ]
+
+        update_user_roles(self.domain_link)
+
+        roles = {r.name: r for r in UserRole.by_domain(self.linked_domain)}
+        self.assertEqual(3, len(roles))
+        self.assertEqual(roles['test'].permissions, remote_permissions)
+        self.assertEqual(roles['test'].is_non_admin_editable, False)
+        self.assertEqual(roles['another'].assignable_by, [self.role1.get_id])
+        self.assertEqual(roles['another'].permissions, Permissions())
+        self.assertEqual(roles['other_test'].assignable_by, [self.role1.get_id])

corehq/apps/linked_domain/updates.py

@@ -235,18 +235,24 @@ def update_user_roles(domain_link):
         upstream_role = copy(role_def)
         upstream_role.pop('_id')
         upstream_role.pop('upstream_id')
+        upstream_role.pop('assignable_by')  # handled below
         role_json.update(upstream_role)
-        local_roles_by_upstream_id[role_json['upstream_id']] = role_json
-        UserRole.wrap(role_json).save()
+        role = UserRole.wrap(role_json)
+        role.save()
+        local_roles_by_upstream_id[role_json['upstream_id']] = role.to_json()
 
     # Update assignable_by ids - must be done after main update to guarantee all local roles have ids
-    for role in local_roles_by_upstream_id.values():
-        if role['assignable_by']:
-            role['assignable_by'] = [
+    for role_def in master_results:
+        role_json = local_roles_by_upstream_id[role_def['_id']]
+        if role_def['assignable_by']:
+            role_json['assignable_by'] = [
                 local_roles_by_upstream_id[role_id]['_id']
-                for role_id in role['assignable_by']
+                for role_id in role_def['assignable_by']
             ]
-            UserRole.wrap(role).save()
+            UserRole.wrap(role_json).save()
+        elif role_json['assignable_by']:
+            role_json['assignable_by'] = None
+            UserRole.wrap(role_json).save()
 
 
 def update_case_search_config(domain_link):

corehq/apps/users/management/commands/populate_user_role.py

@@ -0,0 +1,75 @@
+from corehq.apps.cleanup.management.commands.populate_sql_model_from_couch_model import PopulateSQLCommand
+from corehq.apps.users.models import UserRole, Permissions
+from corehq.apps.users.models_sql import (
+    migrate_role_assignable_by_to_sql,
+    migrate_role_permissions_to_sql,
+    SQLUserRole,
+)
+
+
+class Command(PopulateSQLCommand):
+    @classmethod
+    def couch_db_slug(cls):
+        return "users"
+
+    @classmethod
+    def couch_doc_type(self):
+        return 'UserRole'
+
+    @classmethod
+    def sql_class(self):
+        from corehq.apps.users.models import SQLUserRole
+        return SQLUserRole
+
+    @classmethod
+    def commit_adding_migration(cls):
+        return "TODO: add once the PR adding this file is merged"
+
+    @classmethod
+    def diff_couch_and_sql(cls, couch, sql):
+        diffs = []
+        for field in UserRole._migration_get_fields():
+            diffs.append(cls.diff_attr(field, couch, sql))
+
+        couch_permissions = {
+            info.name: info
+            for info in Permissions.wrap(couch["permissions"]).to_list()
+        }
+        sql_permissions = {
+            info.name: info
+            for info in sql.permissions.to_list()
+        }
+
+        for name in sorted(set(couch_permissions) | set(sql_permissions)):
+            couch_permission = couch_permissions.get(name)
+            diffs.append(cls.diff_attr(
+                "allow",
+                couch_permission._asdict() if couch_permission else {},
+                sql_permissions.get(name),
+                name_prefix=f"permissions.{name}"
+            ))
+
+        couch_assignable_by = couch["assignable_by"]
+        sql_assignable_by = list(sql.roleassignableby_set.values_list('assignable_by_role__couch_id', flat=True))
+        diffs.extend(cls.diff_lists(
+            "assignable_by",
+            sorted(couch_assignable_by) if couch_assignable_by else [],
+            sorted(sql_assignable_by),
+        ))
+        return diffs
+
+    def update_or_create_sql_object(self, doc):
+        model, created = self.sql_class().objects.update_or_create(
+            couch_id=doc['_id'],
+            defaults={
+                "domain": doc.get("domain"),
+                "name": doc.get("name"),
+                "default_landing_page": doc.get("default_landing_page"),
+                "is_non_admin_editable": doc.get("is_non_admin_editable"),
+                "is_archived": doc.get("is_archived"),
+                "upstream_id": doc.get("upstream_id"),
+            })
+        couch_role = UserRole.wrap(doc)
+        migrate_role_permissions_to_sql(couch_role, model)
+        migrate_role_assignable_by_to_sql(couch_role, model)
+        return model, created