Include multi-tenancy in SortingHat

This commit allows to have multiple databases and use each
of them depending on data available in the request.

To configure it you need to set MULTI_TENANT_ENABLED setting
to True, define multiple tenants in `sortinghat/config/tenants.json`
file and assign users to tenants using 'sortinghat-admin set-user-tenant'
command.

Users and tenants relationships will be stored in 'default'
database and other data will be stored in each tenant.

Signed-off-by: Jose Javier Merchante <jjmerchante@bitergia.com>

.github/workflows/release.yml

@@ -76,6 +76,7 @@ jobs:
           PACKAGE=`(cd dist && ls *whl)` && echo $PACKAGE
           pip install --pre ./dist/$PACKAGE
           python manage.py test --settings=config.settings.testing
+          python manage.py test --settings=config.settings.testing_tenant
 
   release:
     needs: [tests]

.github/workflows/tests.yml

@@ -56,6 +56,7 @@ jobs:
     - name: Tests
       run: |
         poetry run python manage.py test --settings=config.settings.testing
+        poetry run python manage.py test --settings=config.settings.testing_tenant
 
   frontend:
 

README.md

@@ -228,6 +228,26 @@ Please update your database running the following command:
 $ sortinghat-admin --config sortinghat.config.settings migrate-old-database
 ```
 
+## Multi-tenancy
+
+SortingHat allows hosting multiple instances with a single service having each
+instance's data isolated in different databases.
+
+To enable this feature follow these guidelines:
+- Set `MULTI_TENANT` settings to `True`.
+- Define a list of tenants using the configuration file `sortinghat/config/tenants.json`. 
+You can use a different json file using the environment variable 
+`SORTINGHAT_MULTI_TENANT_LIST_PATH`
+- Assign users to tenants with the following command:
+  `sortinghat-admin set-user-tenant username host tenant`
+
+There are some limitations:
+- `default` database is only used to store users information and relations between
+users and databases, it won't store anything else related with SortingHat models.
+- Usernames are shared across all instances, which means that it is not possible
+to have the same username with two different passwords in different instances.
+
+
 ## Running tests
 
 SortingHat comes with a comprehensive list of unit tests for both 
@@ -236,6 +256,7 @@ frontend and backend.
 #### Backend test suite
 ```
 (.venv)$ ./manage.py test --settings=config.settings.testing
+(.venv)$ ./manage.py test --settings=config.settings.testing_tenant
 ```
 
 #### Frontend test suite

config/settings/testing.py

@@ -55,6 +55,8 @@
     }
 }
 
+TEST_RUNNER = 'tests.runners.SkipMultiTenantTestRunner'
+
 USE_TZ = True
 
 AUTHENTICATION_BACKENDS = [

config/settings/testing_tenant.py

@@ -0,0 +1,30 @@
+from .testing import *  # noqa: F403,F401
+from .testing import SQL_MODE, DATABASES
+
+
+DATABASES.update({
+    tenant: {
+        'ENGINE': 'django.db.backends.mysql',
+        'USER': 'root',
+        'PASSWORD': 'root',
+        'NAME': tenant,
+        'OPTIONS': {
+            'charset': 'utf8mb4',
+            'sql_mode': ','.join(SQL_MODE)
+        },
+        'TEST': {
+            'NAME': tenant,
+            'CHARSET': 'utf8mb4',
+            'COLLATION': 'utf8mb4_unicode_520_ci',
+        },
+        'HOST': '127.0.0.1',
+        'PORT': 3306
+    }
+    for tenant in ['tenant_1', 'tenant_2']
+})
+
+DATABASE_ROUTERS = [
+    'sortinghat.core.middleware.TenantDatabaseRouter'
+]
+
+TEST_RUNNER = 'tests.runners.OnlyMultiTenantTestRunner'

releases/unreleased/multi-tenancy-mode.yml

@@ -0,0 +1,13 @@
+---
+title: Multi-tenancy mode
+category: added
+author: Jose Javier Merchante <jjmerchante@bitergia.com>
+issue: null
+notes: >
+  SortingHat allows hosting multiple instances with a single service having each
+  instance's data isolated in different databases.
+  
+  To enable this feature follow these guidelines:
+  - Set `MULTI_TENANT` settings to `True`.
+  - Define the tenants in `sortinghat/config/tenants.json`.
+  - Assign users to tenants with `sortinghat-admin set-user-tenant` command.

sortinghat/config/settings.py

@@ -12,10 +12,9 @@
 # https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
 # https://docs.djangoproject.com/en/3.1/ref/settings/
 #
-
+import json
 import os
 
-
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 SILENCED_SYSTEM_CHECKS = [
@@ -123,7 +122,6 @@
     'JWT_ALLOW_ANY_HANDLER': 'sortinghat.core.middleware.allow_any'
 }
 
-
 #
 # Authentication - DO NOT MODIFY
 #
@@ -270,6 +268,40 @@
     }
 }
 
+#
+# SortingHat Multi-tenant
+#
+# To enable this feature:
+#   - Define SORTINGHAT_MULTI_TENANT to True
+#   - Create a list of tenants in sortinghat.config.tenants
+#   - Assign users to tenants with 'set_user_tenant' command.
+#
+
+MULTI_TENANT = os.environ.get('SORTINGHAT_MULTI_TENANT', 'False').lower() in ('true', '1')
+
+if MULTI_TENANT:
+    MIDDLEWARE += ['sortinghat.core.middleware.TenantDatabaseMiddleware']
+    DATABASE_ROUTERS = [
+        'sortinghat.core.middleware.TenantDatabaseRouter'
+    ]
+    MULTI_TENANT_LIST_PATH = os.environ.get('SORTINGHAT_MULTI_TENANT_LIST_PATH',
+                                            os.path.join(BASE_DIR, 'config', 'tenants.json'))
+    with open(MULTI_TENANT_LIST_PATH, 'r') as f:
+        TENANTS_NAMES = json.load(f).get('tenants', [])
+
+    DATABASES.update({
+        tenant: {
+            'ENGINE': 'django.db.backends.mysql',
+            'HOST': os.environ.get('SORTINGHAT_DB_HOST', '127.0.0.1'),
+            'PORT': os.environ.get('SORTINGHAT_DB_PORT', 3306),
+            'USER': os.environ.get('SORTINGHAT_DB_USER', 'root'),
+            'PASSWORD': os.environ.get('SORTINGHAT_DB_PASSWORD', ''),
+            'NAME': tenant,
+            'OPTIONS': {'charset': 'utf8mb4'},
+        }
+        for tenant in TENANTS_NAMES
+    })
+
 #
 # SortingHat workers
 #

sortinghat/config/tenants.json

@@ -0,0 +1,3 @@
+{
+  "tenants": []
+}

sortinghat/core/api.py

@@ -22,8 +22,6 @@
 
 import logging
 
-import django.db.transaction
-
 from grimoirelab_toolkit.datetime import datetime_to_utc
 
 from .db import (find_individual_by_uuid,
@@ -60,13 +58,14 @@
 from .log import TransactionsLog
 from .models import Identity, MIN_PERIOD_DATE, MAX_PERIOD_DATE
 from .aux import merge_datetime_ranges
+from .decorators import atomic_using_tenant
 from ..utils import generate_uuid
 
 
 logger = logging.getLogger(__name__)
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def add_identity(ctx, source, name=None, email=None, username=None, uuid=None):
     """Add an identity to the registry.
 
@@ -157,7 +156,7 @@ def add_identity(ctx, source, name=None, email=None, username=None, uuid=None):
     return identity
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def delete_identity(ctx, uuid):
     """Remove an identity from the registry.
 
@@ -213,7 +212,7 @@ def delete_identity(ctx, uuid):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def update_profile(ctx, uuid, **kwargs):
     """Update individual profile.
 
@@ -265,7 +264,7 @@ def update_profile(ctx, uuid, **kwargs):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def move_identity(ctx, from_uuid, to_uuid):
     """Move an identity to an individual.
 
@@ -340,7 +339,7 @@ def move_identity(ctx, from_uuid, to_uuid):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def lock(ctx, uuid):
     """Lock an individual so it cannot be modified.
 
@@ -374,7 +373,7 @@ def lock(ctx, uuid):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def unlock(ctx, uuid):
     """Unlock an individual so it can be modified.
 
@@ -408,7 +407,7 @@ def unlock(ctx, uuid):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def add_organization(ctx, name):
     """Add an organization to the registry.
 
@@ -445,7 +444,7 @@ def add_organization(ctx, name):
     return org
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def add_domain(ctx, organization, domain_name, is_top_domain=True):
     """Add a domain to the registry.
 
@@ -510,7 +509,7 @@ def add_domain(ctx, organization, domain_name, is_top_domain=True):
     return domain
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def add_team(ctx, team_name, organization=None, parent_name=None):
     """Add a team to the registry.
 
@@ -577,7 +576,7 @@ def add_team(ctx, team_name, organization=None, parent_name=None):
     return team
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def delete_organization(ctx, name):
     """Remove an organization from the registry.
 
@@ -617,7 +616,7 @@ def delete_organization(ctx, name):
     return org
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def delete_domain(ctx, domain_name):
     """Remove a domain from the registry.
 
@@ -656,7 +655,7 @@ def delete_domain(ctx, domain_name):
     return domain
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def delete_team(ctx, team_name, organization=None):
     """Remove a team from the registry.
 
@@ -705,7 +704,7 @@ def delete_team(ctx, team_name, organization=None):
     return team
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def enroll(ctx, uuid, group, parent_org=None, from_date=None, to_date=None,
            force=False):
     """Enroll an individual in a group.
@@ -821,7 +820,7 @@ def enroll(ctx, uuid, group, parent_org=None, from_date=None, to_date=None,
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def withdraw(ctx, uuid, group, parent_org=None, from_date=None, to_date=None):
     """Withdraw an individual from a group.
 
@@ -935,7 +934,7 @@ def withdraw(ctx, uuid, group, parent_org=None, from_date=None, to_date=None):
     return individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def update_enrollment(ctx, uuid, group, from_date, to_date, parent_org=None,
                       new_from_date=None, new_to_date=None, force=True):
     """Update one or more enrollments from an individual given a new date range.
@@ -1024,7 +1023,7 @@ def update_enrollment(ctx, uuid, group, from_date, to_date, parent_org=None,
     return indv
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def merge(ctx, from_uuids, to_uuid):
     """
     Merge one or more individuals into another.
@@ -1197,7 +1196,7 @@ def _delete_individuals(trxl, individuals):
     return to_individual
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def unmerge_identities(ctx, uuids):
     """
     Unmerge one or more identities from their corresponding individual.
@@ -1293,7 +1292,7 @@ def _move_to_destination(trxl, identity, individual):
     return new_individuals
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def delete_import_identities_task(ctx, task_id):
     """Remove an import identities task from the registry.
 
@@ -1327,7 +1326,7 @@ def delete_import_identities_task(ctx, task_id):
     return task
 
 
-@django.db.transaction.atomic
+@atomic_using_tenant
 def update_import_identities_task(ctx, task_id, **kwargs):
     """Update an import identities task.
 

sortinghat/core/context.py

@@ -24,6 +24,6 @@
 
 
 SortingHatContext = collections.namedtuple(
-    'SortingHatContext', ['user', 'job_id']
+    'SortingHatContext', ['user', 'job_id', 'tenant']
 )
-SortingHatContext.__new__.__defaults__ = (None, None)
+SortingHatContext.__new__.__defaults__ = (None, None, 'default')