From bbb4810350d6212660760082a89e260507d81342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=BCrkan=20=C4=B0ndibay?= Date: Tue, 28 Feb 2023 00:40:24 +0300 Subject: [PATCH] Adds readthedocs documents and integration (#148) --- .readthedocs.yaml | 15 + .vscode/settings.json | 1 + README.md | 3 +- docs/Makefile | 20 + docs/requirements.in | 4 + docs/requirements.txt | 70 ++++ docs/source/conf.py | 30 ++ docs/source/general.rst | 17 + docs/source/index.rst | 53 +++ docs/source/license.rst | 20 + docs/source/migration_mt_django.rst | 559 ++++++++++++++++++++++++++++ docs/source/usage.rst | 237 ++++++++++++ 12 files changed, 1028 insertions(+), 1 deletion(-) create mode 100644 .readthedocs.yaml create mode 100644 docs/Makefile create mode 100644 docs/requirements.in create mode 100644 docs/requirements.txt create mode 100644 docs/source/conf.py create mode 100644 docs/source/general.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/license.rst create mode 100644 docs/source/migration_mt_django.rst create mode 100644 docs/source/usage.rst diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..823a3f3f --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,15 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + +# Build from the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + +# Explicitly set the version of Python and its requirements +python: + install: + - requirements: docs/requirements.txt \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7e10ba3a..747b2f28 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,4 +4,5 @@ "python.envFile": "${workspaceFolder}/.envs", "python.linting.prospectorEnabled": true, "python.linting.enabled": true, + "esbonio.sphinx.confDir": "", } diff --git a/README.md b/README.md index 16c63e5f..57dba8b6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# django-multitenant [![Build Status](https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml/badge.svg)](https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml) +# django-multitenant
[![Build Status](https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml/badge.svg)](https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml) [![Latest Documentation Status](https://readthedocs.org/projects/django-multitenant/badge/?version=latest)](https://django-multitenant.readthedocs.io/en/latest/?badge=latest) [![Coverage Status](https://codecov.io/gh/citusdata/django-multitenant/branch/main/graph/badge.svg?token=taRgoSgHUw)](https://codecov.io/gh/citusdata/django-multitenant) [![PyPI Version](https://badge.fury.io/py/django-multitenant.svg)](https://badge.fury.io/py/django-multitenant) + Python/Django support for distributed multi-tenant databases like Postgres+Citus Enables easy scale-out by adding the tenant context to your queries, enabling the database (e.g. Citus) to efficiently route queries to the right database node. diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..d0c3cbf1 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 00000000..da3e99a3 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,4 @@ +sphinxnotes-strike +sphinx +sphinx_rtd_theme +readthedocs-sphinx-search \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..586f31a0 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,70 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --output-file=requirements.txt --resolver=backtracking requirements.in +# +alabaster==0.7.13 + # via sphinx +babel==2.11.0 + # via sphinx +certifi==2022.12.7 + # via requests +charset-normalizer==3.0.1 + # via requests +docutils==0.18.1 + # via + # sphinx + # sphinx-rtd-theme +idna==3.4 + # via requests +imagesize==1.4.1 + # via sphinx +importlib-metadata==6.0.0 + # via sphinx +jinja2==3.1.2 + # via sphinx +markupsafe==2.1.2 + # via jinja2 +packaging==23.0 + # via sphinx +pygments==2.14.0 + # via sphinx +pytz==2022.7.1 + # via babel +readthedocs-sphinx-search==0.2.0 + # via -r requirements.in +requests==2.28.2 + # via sphinx +snowballstemmer==2.2.0 + # via sphinx +sphinx==6.1.3 + # via + # -r requirements.in + # sphinx-rtd-theme + # sphinxnotes-strike +sphinx-rtd-theme==1.2.0 + # via -r requirements.in +sphinxcontrib-applehelp==1.0.4 + # via sphinx +sphinxcontrib-devhelp==1.0.2 + # via sphinx +sphinxcontrib-htmlhelp==2.0.1 + # via sphinx +sphinxcontrib-jquery==2.0.0 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==1.0.3 + # via sphinx +sphinxcontrib-serializinghtml==1.1.5 + # via sphinx +sphinxnotes-strike==1.2 + # via -r requirements.in +urllib3==1.26.14 + # via requests +zipp==3.13.0 + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..c7dccc84 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,30 @@ +from datetime import date + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Django Multi-tenant" +copyright = f"{date.today().year} .Citus Data Licensed under the MIT license, see License for details. " +author = "Citus Data" +release = "3.0.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = ["sphinxnotes.strike"] + +templates_path = ["_templates"] +exclude_patterns = [] + +language = "python" + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_static_path = ["_static"] diff --git a/docs/source/general.rst b/docs/source/general.rst new file mode 100644 index 00000000..452ef53c --- /dev/null +++ b/docs/source/general.rst @@ -0,0 +1,17 @@ +.. _general: + +Installation +================================= + +1. ``pip install --no-cache-dir django_multitenant`` + +Supported Django and Citus versions/Pre-requisites +=================================================== + +================= ====== ===== +Python Django Citus +================= ====== ===== +3.7 3.2 10 11 +3.8 3.9 3.10 3.11 4.0 10 11 +3.8 3.9 3.10 3.11 4.1 11 +================= ====== ===== \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..fe9daddb --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,53 @@ +.. Django Multi-tenant documentation master file, created by + sphinx-quickstart on Mon Feb 13 13:32:28 2023. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to Django Multi-tenant's documentation! +================================================= + +|Latest Documentation Status| |Build Status| |Coverage Status| |PyPI Version| + +.. |Latest Documentation Status| image:: https://readthedocs.org/projects/django-multitenant/badge/?version=latest + :target: https://django-multitenant.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status + +.. |Build Status| image:: https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml/badge.svg + :target: https://github.com/citusdata/django-multitenant/actions/workflows/django-multitenant-tests.yml + :alt: Build Status + +.. |Coverage Status| image:: https://codecov.io/gh/citusdata/django-multitenant/branch/main/graph/badge.svg?token=taRgoSgHUw + :target: https://codecov.io/gh/citusdata/django-multitenant + :alt: Coverage Status + +.. |PyPI Version| image:: https://badge.fury.io/py/django-multitenant.svg + :target: https://badge.fury.io/py/django-multitenant + + +Python/Django support for distributed multi-tenant databases like +Postgres+Citus + +Enables easy scale-out by adding the tenant context to your queries, +enabling the database (e.g. Citus) to efficiently route queries to the +right database node. + +There are architecures for building multi-tenant databases viz. **Create +one database per tenant**, **Create one schema per tenant** and **Have +all tenants share the same table(s)**. This library is based on the 3rd +design i.e **Have all tenants share the same table(s)**, it assumes that +all the tenant relates models/tables have a tenant_id column for +representing a tenant. + +The following link talks more about the trade-offs on when and how to +choose the right architecture for your multi-tenat database: + +https://www.citusdata.com/blog/2016/10/03/designing-your-saas-database-for-high-scalability/ + +.. toctree:: + :maxdepth: 2 + :caption: Table Of Contents + + general + usage + migration_mt_django + license diff --git a/docs/source/license.rst b/docs/source/license.rst new file mode 100644 index 00000000..0a4ad717 --- /dev/null +++ b/docs/source/license.rst @@ -0,0 +1,20 @@ +License +=============================================== +Copyright (c) 2023 , Citus Data Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the +Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/docs/source/migration_mt_django.rst b/docs/source/migration_mt_django.rst new file mode 100644 index 00000000..d08eeb95 --- /dev/null +++ b/docs/source/migration_mt_django.rst @@ -0,0 +1,559 @@ + +.. _django_migration: + +Migrating a multi-tenant Django application to Citus +===================================================== + +Here we investigate specifically how to migrate multi-tenant Django applications to a Citus storage backend with the help of the `django-multitenant `_ library. + +This process will be in 5 steps: + +- Introducing the tenant column to models missing it that we want to distribute +- Changing the primary keys of distributed tables to include the tenant column +- Updating the models to use the :code:`TenantModelMixin` +- Distributing the data +- Updating the Django Application to scope queries + +Preparing to scale-out a multi-tenant application +-------------------------------------------------- + +Initially you’ll start with all tenants placed on a single database node. To be able to scale out django, some simple changes will have to be made to your models. + +Let's consider this simplified model: + +.. code-block:: python + + from django.utils import timezone + from django.db import models + + class Country(models.Model): + name = models.CharField(max_length=255) + + class Account(models.Model): + name = models.CharField(max_length=255) + domain = models.CharField(max_length=255) + subdomain = models.CharField(max_length=255) + country = models.ForeignKey(Country, on_delete=models.SET_NULL) + + class Manager(models.Model): + name = models.CharField(max_length=255) + account = models.ForeignKey(Account, on_delete=models.CASCADE, + related_name='managers') + + class Project(models.Model): + name = models.CharField(max_length=255) + account = models.ForeignKey(Account, related_name='projects', + on_delete=models.CASCADE) + managers = models.ManyToManyField(Manager) + + class Task(models.Model): + name = models.CharField(max_length=255) + project = models.ForeignKey(Project, on_delete=models.CASCADE, + related_name='tasks') + +The tricky thing with this pattern is that in order to find all tasks for an account, you'll have to query for all of an account's project first. This becomes a problem once you start sharding data, and in particular when you run UPDATE or DELETE queries on nested models like task in this example. + +1. Introducing the tenant column to models belonging to an account +------------------------------------------------------------------ + +**1.1 Introducing the column to models belonging to an account** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In order to scale out a multi-tenant model, it’s essential for queries to quickly +locate all records that belong to an account. Consider an ORM call such as: + +.. code-block:: python + + Project.objects.filter(account_id=1).prefetch_related('tasks') + +It generates these underlying SQL queries: + +.. code-block:: postgresql + + SELECT * + FROM myapp_project + WHERE account_id = 1; + + SELECT * + FROM myapp_task + WHERE project_id IN (1, 2, 3); + +However, the second query would go faster with an extra filter: + +.. code-block:: postgresql + + -- the AND clause identifies the tenant + SELECT * + FROM myapp_task + WHERE project_id IN (1, 2, 3) + AND account_id = 1; + +This way you can easily query the tasks belonging to one account. +The easiest way to achieve this is to simply add a :code:`account_id` column on every object that belongs to an account. + +In our case: + +.. code-block:: python + + class Task(models.Model): + name = models.CharField(max_length=255) + project = models.ForeignKey(Project, on_delete=models.CASCADE, + related_name='tasks') + account = models.ForeignKey(Account, related_name='tasks', + on_delete=models.CASCADE) + +Create a migration to reflect the change: :code:`python manage.py makemigrations`. + +**1.2. Introduce a column for the account\_id on every ManyToMany model that belongs to an account** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The goal is the same as previously. We want to be able to have ORM calls and queries routed to one account. We also want to be able to distribute the ManyToMany relationship related to an account on the account_id. + +So the calls generated by: + +.. code-block:: python + + Project.objects.filter(account_id=1).prefetch_related('managers') + +Can include in their :code:`WHERE` clause the :code:`account_id` like this: + +.. code-block:: postgresql + + SELECT * + FROM "myapp_project" WHERE "myapp_project"."account_id" = 1; + + SELECT * + FROM myapp_manager manager + INNER JOIN myapp_projectmanager projectmanager + ON (manager.id = projectmanager.manager_id + AND projectmanager.account_id = manager.account_id) + WHERE projectmanager.project_id IN (1, 2, 3) + AND manager.account_id = 1; + +For that we need to introduce :code:`through` models. In our case: + +.. code-block:: python + + class Project(models.Model): + name = models.CharField(max_length=255) + account = models.ForeignKey(Account, related_name='projects', + on_delete=models.CASCADE) + managers = models.ManyToManyField(Manager, through='ProjectManager') + + class ProjectManager(models.Model): + project = models.ForeignKey(Project, on_delete=models.CASCADE) + manager = models.ForeignKey(Manager, on_delete=models.CASCADE) + account = models.ForeignKey(Account, on_delete=models.CASCADE) + +Create a migration to reflect the change: :code:`python manage.py makemigrations`. + +2. Include the account\_id in all primary keys and unique constraints +--------------------------------------------------------------------- + +Primary-key and unique constraints on values other than the tenant\_id +will present a problem in any distributed system, since it’s difficult +to make sure that no two nodes accept the same unique value. Enforcing +the constraint would require expensive scans of the data across all +nodes. + +To solve this problem, for the models which are logically related +to an account (the tenant for our app), you should add account\_id to +the primary keys and unique constraints, effectively scoping objects unique inside a given +account. This helps add the concept of tenancy to your models, thereby +making the multi-tenant system more robust. + +**2.1 Including the account\_id to primary keys** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Django automatically creates a simple "id" primary key on models, so we will need to circumvent that behavior with a custom migration of our own. Run :code:`python manage.py makemigrations appname --empty --name remove_simple_pk`, and edit the result to look like this: + +.. code-block:: python + + from django.db import migrations + + class Migration(migrations.Migration): + + dependencies = [ + # leave this as it was generated + ] + + operations = [ + # Django considers "id" the primary key of these tables, but + # we want the primary key to be (account_id, id) + migrations.RunSQL(""" + ALTER TABLE myapp_manager + DROP CONSTRAINT myapp_manager_pkey CASCADE; + + ALTER TABLE myapp_manager + ADD CONSTRAINT myapp_manager_pkey + PRIMARY KEY (account_id, id); + """), + + migrations.RunSQL(""" + ALTER TABLE myapp_project + DROP CONSTRAINT myapp_project_pkey CASCADE; + + ALTER TABLE myapp_project + ADD CONSTRAINT myapp_product_pkey + PRIMARY KEY (account_id, id); + """), + + migrations.RunSQL(""" + ALTER TABLE myapp_task + DROP CONSTRAINT myapp_task_pkey CASCADE; + + ALTER TABLE myapp_task + ADD CONSTRAINT myapp_task_pkey + PRIMARY KEY (account_id, id); + """), + + migrations.RunSQL(""" + ALTER TABLE myapp_projectmanager + DROP CONSTRAINT myapp_projectmanager_pkey CASCADE; + + ALTER TABLE myapp_projectmanager + ADD CONSTRAINT myapp_projectmanager_pkey PRIMARY KEY (account_id, id); + """), + ] + +**2.2 Including the account\_id to unique constraints** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The same thing needs to be done for ``UNIQUE`` constraints. You can have explicit constraints that you might have set in your model with ``unique=True`` or ``unique_together`` like: + +.. code-block:: python + + class Project(models.Model): + name = models.CharField(max_length=255, unique=True) + account = models.ForeignKey(Account, related_name='projects', + on_delete=models.CASCADE) + managers = models.ManyToManyField(Manager, through='ProjectManager') + + class Task(models.Model): + name = models.CharField(max_length=255) + project = models.ForeignKey(Project, on_delete=models.CASCADE, + related_name='tasks') + account = models.ForeignKey(Account, related_name='tasks', + on_delete=models.CASCADE) + + class Meta: + unique_together = [('name', 'project')] + +For these constraints, you can simply change in the models the constraints: + +.. code-block:: python + + class Project(models.Model): + name = models.CharField(max_length=255) + account = models.ForeignKey(Account, related_name='projects', + on_delete=models.CASCADE) + managers = models.ManyToManyField(Manager, through='ProjectManager') + + class Meta: + unique_together = [('account', 'name')] + + class Task(models.Model): + name = models.CharField(max_length=255) + project = models.ForeignKey(Project, on_delete=models.CASCADE, + related_name='tasks') + account = models.ForeignKey(Account, related_name='tasks', + on_delete=models.CASCADE) + + class Meta: + unique_together = [('account', 'name', 'project')] + +Then generate the migration with: + +.. code-block:: python + + python manage.py makemigrations + +Some ``UNIQUE`` constraints are created by the ORM and you will need to explicitly drop them. +This is the case for ``OneToOneField`` and ``ManyToMany`` fields. + +For these cases you will need to: +1. Find the constraints +2. Do a migration to drop them +3. Re-create constraints including the account\_id field + +To find the constraints, connect to your database using ``psql`` and run ``\d+ myapp_projectmanager`` +You will see the ``ManyToMany`` (or ``OneToOneField``) constraint: + +.. code-block:: sql + + "myapp_projectmanager" UNIQUE CONSTRAINT myapp_projectman_project_id_manager_id_bc477b48_uniq, + btree (project_id, manager_id) + +Drop this constraint in a migration: + +.. code-block:: python + + from django.db import migrations + + class Migration(migrations.Migration): + + dependencies = [ + # leave this as it was generated + ] + + operations = [ + migrations.RunSQL(""" + DROP CONSTRAINT myapp_projectman_project_id_manager_id_bc477b48_uniq; + """), + +Then change your models to have a ``unique_together`` including the ``account\_id`` + +.. code-block:: python + + class ProjectManager(models.Model): + project = models.ForeignKey(Project, on_delete=models.CASCADE) + manager = models.ForeignKey(Manager, on_delete=models.CASCADE) + account = models.ForeignKey(Account, on_delete=models.CASCADE) + + class Meta: + unique_together=(('account', 'project', 'manager')) + +And finally apply the changes by creating a new migration to generate these constraints: + +.. code-block:: python + + python manage.py makemigrations + +3. Updating the models to use TenantModelMixin and TenantForeignKey +-------------------------------------------------------------------- + +Next, we'll use the `django-multitenant `_ library to add account_id to foreign keys, and make application queries easier later on. + +In requirements.txt for your Django application, add + +:: + + django_multitenant>=2.0.0, <3 + +Run ``pip install -r requirements.txt``. + +In settings.py, change the database engine to the customized engine provided by django-multitenant: + +.. code-block:: python + + 'ENGINE': 'django_multitenant.backends.postgresql' + +**3.1 Introducing the TenantModelMixin and TenantManager** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The models will now not only inherit from ``models.Model`` but also from the ``TenantModelMixin``. + +To do that in your :code:`models.py` file you will need to do the following imports + +.. code-block:: python + + from django_multitenant.mixins import * + +Previously our example models inherited from just models.Model, but now we need +to change them to also inherit from TenantModelMixin. The models in real +projects may inherit from other mixins too like ``django.contrib.gis.db``, +which is fine. + +You will also, at this point, introduce the tenant_id to define which column is +the distribution column. + +.. code-block:: python + + class TenantManager(TenantManagerMixin, models.Manager): + pass + + class Account(TenantModelMixin, models.Model): + ... + tenant_id = 'id' + objects = TenantManager() + + class Manager(TenantModelMixin, models.Model): + ... + tenant_id = 'account_id' + objects = TenantManager() + + class Project(TenantModelMixin, models.Model): + ... + tenant_id = 'account_id' + objects = TenantManager() + + class Task(TenantModelMixin, models.Model): + ... + tenant_id = 'account_id' + objects = TenantManager() + + class ProjectManager(TenantModelMixin, models.Model): + ... + tenant_id = 'account_id' + objects = TenantManager() + +**3.2 Handling ForeignKey constraints** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For ``ForeignKey`` and ``OneToOneField`` constraint, we have a few different cases: + +- Foreign keys (or One to One) between distributed tables, for which you should use the ``TenantForeignKey`` (or ``TenantOneToOneField``). +- Foreign keys between a distributed table and a reference table don't require a change. +- Foreign keys between a distributed table and a local table, which require to drop the constraint by using ``models.ForeignKey(MyModel, on_delete=models.CASCADE, db_constraint=False)``. + +Finally your models should look like this: + +.. code-block:: python + + from django.db import models + from django_multitenant.fields import TenantForeignKey + from django_multitenant.mixins import * + + class Country(models.Model): # This table is a reference table + name = models.CharField(max_length=255) + + class TenantManager(TenantManagerMixin, models.Manager): + pass + + class Account(TenantModelMixin, models.Model): + name = models.CharField(max_length=255) + domain = models.CharField(max_length=255) + subdomain = models.CharField(max_length=255) + country = models.ForeignKey(Country, on_delete=models.SET_NULL) # No changes needed + + tenant_id = 'id' + objects = TenantManager() + + class Manager(TenantModelMixin, models.Model): + name = models.CharField(max_length=255) + account = models.ForeignKey(Account, related_name='managers', + on_delete=models.CASCADE) + tenant_id = 'account_id' + objects = TenantManager() + + class Project(TenantModelMixin, models.Model): + account = models.ForeignKey(Account, related_name='projects', + on_delete=models.CASCADE) + managers = models.ManyToManyField(Manager, through='ProjectManager') + tenant_id = 'account_id' + objects = TenantManager() + + class Task(TenantModelMixin, models.Model): + name = models.CharField(max_length=255) + project = TenantForeignKey(Project, on_delete=models.CASCADE, + related_name='tasks') + account = models.ForeignKey(Account, on_delete=models.CASCADE) + + tenant_id = 'account_id' + objects = TenantManager() + + class ProjectManager(TenantModelMixin, models.Model): + project = TenantForeignKey(Project, on_delete=models.CASCADE) + manager = TenantForeignKey(Manager, on_delete=models.CASCADE) + account = models.ForeignKey(Account, on_delete=models.CASCADE) + + tenant_id = 'account_id' + objects = TenantManager() + +**3.3 Handling ManyToMany constraints** +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the second section of this article, we introduced the fact that with citus, ``ManyToMany`` relationships require a ``through`` model with the tenant column. Which is why we have the model: + +.. code-block:: python + + class ProjectManager(TenantModelMixin, models.Model): + project = TenantForeignKey(Project, on_delete=models.CASCADE) + manager = TenantForeignKey(Manager, on_delete=models.CASCADE) + account = models.ForeignKey(Account, on_delete=models.CASCADE) + + tenant_id = 'account_id' + objects = TenantManager() + +After installing the library, changing the engine, and updating the models, run +:code:`python manage.py makemigrations`. This will produce a migration to make the foreign keys composite when necessary. + +4. Distribute data in Citus +---------------------------- + +We need one final migration to tell Citus to mark tables for distribution. Create a new migration :code:`python manage.py makemigrations appname --empty --name distribute_tables`. Edit the result to look like this: + +.. code-block:: python + + from django.db import migrations + from django_multitenant.db import migrations as tenant_migrations + + class Migration(migrations.Migration): + dependencies = [ + # leave this as it was generated + ] + + operations = [ + tenant_migrations.Distribute('Country', reference=True), + tenant_migrations.Distribute('Account'), + tenant_migrations.Distribute('Manager'), + tenant_migrations.Distribute('Project'), + tenant_migrations.Distribute('ProjectManager'), + tenant_migrations.Distribute('Task'), + ] + +With all the migrations created from the steps so far, apply them to the database with ``python manage.py migrate``. + +At this point the Django application models are ready to work with a Citus backend. You can continue by importing data to the new system and modifying views as necessary to deal with the model changes. + +Updating the Django Application to scope queries +------------------------------------------------ + +The django-multitenant library discussed in the previous section is not only useful for migrations, but also for simplifying application queries. The library allows application code to easily scope queries to a single tenant. It automatically adds the correct SQL filters to all statements, including fetching objects through relations. + +For instance, in a view simply ``set_current_tenant`` and all the queries or joins afterward will include a filter to scope results to a single tenant. + +.. code-block:: python + + # set the current tenant to the first account + s = Account.objects.first() + set_current_tenant(s) + + # now this count query applies only to Project for that account + Project.objects.count() + + # Find tasks for very important projects in the current account + Task.objects.filter(project__name='Very important project') + +In the context of an application view, the current tenant object can be stored as a SESSION variable when a user logs in, and view actions can :code:`set_current_tenant` to this value. See the README in django-multitenant for more examples. + +The ``set_current_tenant`` function can also take an array of objects, like + +.. code-block:: python + + set_current_tenant([s1, s2, s3]) + +which updates the internal SQL query with a filter like ``tenant_id IN (a,b,c)``. + +Automating with middleware +-------------------------- + +Rather than calling ``set_current_tenant()`` in each view, you can create and install a new `middleware `_ class in your Django application to do it automatically. + +.. code-block:: python + + # src/appname/middleware.py + + from django_multitenant.utils import set_current_tenant + + class MultitenantMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if request.user and not request.user.is_anonymous: + set_current_tenant(request.user.employee.company) + response = self.get_response(request) + return response + +Enable the middleware by updating the MIDDLEWARE array in src/appname/settings/base.py: + +.. code-block:: python + + MIDDLEWARE = [ + # ... + # existing items + # ... + + 'appname.middleware.MultitenantMiddleware' + ] diff --git a/docs/source/usage.rst b/docs/source/usage.rst new file mode 100644 index 00000000..0b62eb92 --- /dev/null +++ b/docs/source/usage.rst @@ -0,0 +1,237 @@ +Usage +================================= + +In order to use this library you can either use Mixins or have your +models inherit from our custom model class. + +Changes in Models: +------------------ + +1. In whichever files you want to use the library import it: + + .. code:: python + + from django_multitenant.fields import * + from django_multitenant.models import * + +2. All models should inherit the TenantModel class. + ``Ex: class Product(TenantModel):`` + +3. Define a static variable named tenant_id and specify the tenant + column using this variable. ``Ex: tenant_id='store_id'`` + +4. All foreign keys to TenantModel subclasses should use + TenantForeignKey in place of models.ForeignKey + +5. A sample model implementing the above 2 steps: + + .. code:: python + + class Store(TenantModel): + tenant_id = 'id' + name = models.CharField(max_length=50) + address = models.CharField(max_length=255) + email = models.CharField(max_length=50) + + class Product(TenantModel): + store = models.ForeignKey(Store) + tenant_id='store_id' + name = models.CharField(max_length=255) + description = models.TextField() + class Meta(object): + unique_together = ["id", "store"] + class Purchase(TenantModel): + store = models.ForeignKey(Store) + tenant_id='store_id' + product_purchased = TenantForeignKey(Product) + + +Reserved tenant_id keyword +~~~~~~~~~~~~~~~~~~~~~~~~~~ +tenant_id column name should not be 'tenant_id'. 'tenant_id' is a reserved keyword across the library. + +Example model with correct tenant_id column name: + +.. code:: python + + class Tenant + tenant_id = 'id' + + class Business(TenantModel): + ten = models.ForeignKey(Tenant, blank=True, null=True, on_delete=models.SET_NULL) + tenant_id = 'tenant_id' # This is wrong + tenant_id = 'ten_id' # This is correct + + +Changes in Models using mixins +------------------------------- + +1. In whichever files you want to use the library import it by just + saying + + .. code:: python + + from django_multitenant.mixins import * + +2. All models should use the ``TenantModelMixin`` and the django + ``models.Model`` or your customer Model class + ``Ex: class Product(TenantModelMixin, models.Model):`` + +3. Define a static variable named tenant_id and specify the tenant + column using this variable. ``Ex: tenant_id='store_id'`` + +4. All foreign keys to TenantModel subclasses should use + TenantForeignKey in place of models.ForeignKey + +5. Referenced table in TenenatForeignKey should include a unique key + including tenant_id and primary key + + :: + + Ex: + class Meta(object): + unique_together = ["id", "store"] + +6. A sample model implementing the above 3 steps: + + .. code:: python + + + class ProductManager(TenantManagerMixin, models.Manager): + pass + + class Product(TenantModelMixin, models.Model): + store = models.ForeignKey(Store) + tenant_id='store_id' + name = models.CharField(max_length=255) + description = models.TextField() + + objects = ProductManager() + + class Meta(object): + unique_together = ["id", "store"] + + class PurchaseManager(TenantManagerMixin, models.Manager): + pass + + class Purchase(TenantModelMixin, models.Model): + store = models.ForeignKey(Store) + tenant_id='store_id' + product_purchased = TenantForeignKey(Product) + + objects = PurchaseManager() + +Automating composite foreign keys at db layer +---------------------------------------------- + +1. Creating foreign keys between tenant related models using + TenantForeignKey would automate adding tenant_id to reference queries + (ex. product.purchases) and join queries (ex. product__name). If you + want to ensure to create composite foreign keys (with tenant_id) at + the db layer, you should change the database ENGINE in the + settings.py to ``django_multitenant.backends.postgresql``. + + .. code:: python + + 'default': { + 'ENGINE': 'django_multitenant.backends.postgresql', + ...... + ...... + ...... + } + +Where to Set the Tenant? +------------------------ + +1. Write authentication logic using a middleware which also sets/unsets + a tenant for each session/request. This way developers need not worry + about setting a tenant on a per view basis. Just set it while + authentication and the library would ensure the rest (adding + tenant_id filters to the queries). A sample implementation of the + above is as follows: + + .. code:: python + + from django_multitenant.utils import set_current_tenant + + class MultitenantMiddleware: + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + if request.user and not request.user.is_anonymous: + set_current_tenant(request.user.employee.company) + return self.get_response(request) + + In your settings, you will need to update the ``MIDDLEWARE`` setting + to include the one you created. + + .. code:: python + + MIDDLEWARE = [ + # ... + # existing items + # ... + 'appname.middleware.MultitenantMiddleware' + ] + +2. Set the tenant using set_current_tenant(t) api in all the views which + you want to be scoped based on tenant. This would scope all the + django API calls automatically(without specifying explicit filters) + to a single tenant. If the current_tenant is not set, then the + default/native API without tenant scoping is used. + + .. code:: python + + def application_function: + # current_tenant can be stored as a SESSION variable when a user logs in. + # This should be done by the app + t = current_tenant + #set the tenant + set_current_tenant(t); + #Django ORM API calls; + #Command 1; + #Command 2; + #Command 3; + #Command 4; + #Command 5; + +Supported APIs +================================= + +1. Most of the APIs under Model.objects.*. +2. Model.save() injects tenant_id for tenant inherited models. + +.. code:: python + + s=Store.objects.all()[0] + set_current_tenant(s) + + #All the below API calls would add suitable tenant filters. + #Simple get_queryset() + Product.objects.get_queryset() + + #Simple join + Purchase.objects.filter(id=1).filter(store__name='The Awesome Store').filter(product__description='All products are awesome') + + #Update + Purchase.objects.filter(id=1).update(id=1) + + #Save + p=Product(8,1,'Awesome Shoe','These shoes are awesome') + p.save() + + #Simple aggregates + Product.objects.count() + Product.objects.filter(store__name='The Awesome Store').count() + + #Subqueries + Product.objects.filter(name='Awesome Shoe'); + Purchase.objects.filter(product__in=p); + +Credits +================================= + +This library uses similar logic of setting/getting tenant object as in +`django-simple-multitenant `__. +We thank the authors for their efforts.