-
-
Notifications
You must be signed in to change notification settings - Fork 7.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
sessions: Migrate to custom indexed session model.
Fixes #19490 and has other performance benefits. Create RealmSession model with custom fields: realm, user and ip_address, indexed by (realm, user) and ip_address. Migrate code to use that model, "zerver/lib/sessions" has most impact. Create data_migration to copy over data from django_session to zerver_realmsession table.
- Loading branch information
1 parent
7104c06
commit 2edb41e
Showing
19 changed files
with
417 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
# Generated by Django 4.2.11 on 2024-04-17 04:28 | ||
|
||
import django.db.models.deletion | ||
from django.conf import settings | ||
from django.db import migrations, models | ||
|
||
|
||
class Migration(migrations.Migration): | ||
dependencies = [ | ||
( | ||
"zerver", | ||
"0515_rename_named_group_can_mention_group_namedusergroup_can_mention_group_and_more", | ||
), | ||
] | ||
|
||
operations = [ | ||
migrations.CreateModel( | ||
name="RealmSession", | ||
fields=[ | ||
( | ||
"session_key", | ||
models.CharField( | ||
max_length=40, primary_key=True, serialize=False, verbose_name="session key" | ||
), | ||
), | ||
("session_data", models.TextField(verbose_name="session data")), | ||
("expire_date", models.DateTimeField(db_index=True, verbose_name="expire date")), | ||
("ip_address", models.GenericIPAddressField(null=True)), | ||
( | ||
"realm", | ||
models.ForeignKey( | ||
null=True, on_delete=django.db.models.deletion.CASCADE, to="zerver.realm" | ||
), | ||
), | ||
( | ||
"user", | ||
models.ForeignKey( | ||
null=True, | ||
on_delete=django.db.models.deletion.CASCADE, | ||
to=settings.AUTH_USER_MODEL, | ||
), | ||
), | ||
], | ||
options={ | ||
"indexes": [ | ||
models.Index( | ||
fields=["realm", "user"], name="zerver_realmsession_realm_user_idx" | ||
), | ||
models.Index(fields=["ip_address"], name="zerver_realmsession_ip_address_idx"), | ||
], | ||
}, | ||
), | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
# Generated by Django 4.2.11 on 2024-04-17 04:30 | ||
|
||
from django.contrib.sessions.models import Session | ||
from django.db import connection, migrations, transaction | ||
from django.db.backends.base.schema import BaseDatabaseSchemaEditor | ||
from django.db.migrations.state import StateApps | ||
from psycopg2.sql import SQL, Composable, Identifier | ||
|
||
# This migration copies over data from existing sessions table "django_session" | ||
# to the custom session table "zerver_realmsession" which is introduced in previous migration. | ||
|
||
# Since "django_session" is a large table we want to do the insertion as performantly as possible, | ||
# approach is pretty simple: drop all indexes, copy data, create back the indexes, | ||
# because creating an index on pre-existing data is quicker than updating it incrementally as each row is loaded. | ||
|
||
# Both drop indexes and copy data are done in one transaction | ||
# (because we can't write to postgres then update its scheme (i.e. create the indexes) in the same transaction), | ||
# so creating back the indexes should be in a separate transaction. | ||
|
||
|
||
def migrate_data_from_django_session_to_realmsession( | ||
apps: StateApps, schema_editor: BaseDatabaseSchemaEditor | ||
) -> None: | ||
# Just in case django_session table was empty, then no need to do all of this. | ||
if not Session.objects.exists(): | ||
return | ||
|
||
session_key_index_name = "" | ||
with transaction.atomic(): | ||
with connection.cursor() as cursor: | ||
# Fetch indexes names. | ||
cursor.execute( | ||
SQL("""SELECT indexname FROM pg_indexes WHERE tablename = 'zerver_realmsession' """) | ||
) | ||
|
||
indexes_tuples = cursor.fetchall() | ||
# Drop all indexes. | ||
for index in indexes_tuples: | ||
if index[0] == "zerver_realmsession_pkey": | ||
query: Composable = SQL( | ||
"""ALTER TABLE zerver_realmsession drop constraint zerver_realmsession_pkey""" | ||
) | ||
else: | ||
if "session_key" in index[0]: | ||
session_key_index_name = index[0] | ||
|
||
query = SQL("""DROP INDEX {index_name}""").format( | ||
index_name=Identifier(index[0]) | ||
) | ||
|
||
cursor.execute(query) | ||
|
||
# Copy over data. | ||
cursor.execute(SQL("INSERT INTO zerver_realmsession (SELECT * FROM django_session)")) | ||
|
||
# If Insertion is successful, Should we clear django_session table ? | ||
# Session.objects.all().delete() | ||
|
||
# Create back the indexes | ||
with transaction.atomic(): | ||
with connection.cursor() as cursor: | ||
# Primary key creates an index by default. | ||
cursor.execute( | ||
SQL( | ||
""" | ||
ALTER TABLE zerver_realmsession ADD CONSTRAINT | ||
zerver_realmsession_pkey PRIMARY KEY (session_key) | ||
""" | ||
) | ||
) | ||
|
||
cursor.execute( | ||
SQL( | ||
# {index_name} was auto generated by django, we create the index with the same name just in case, | ||
# as i'm not really sure if a custom name in this case would be a problem. | ||
""" | ||
CREATE INDEX IF NOT EXISTS {index_name} | ||
ON zerver_realmsession (session_key varchar_pattern_ops); | ||
CREATE INDEX IF NOT EXISTS zerver_realmsession_expire_date_idx | ||
ON zerver_realmsession (expire_date); | ||
CREATE INDEX IF NOT EXISTS zerver_realmsession_realm_idx | ||
ON zerver_realmsession (realm_id); | ||
CREATE INDEX IF NOT EXISTS zerver_realmsession_user_idx | ||
ON zerver_realmsession (user_id); | ||
CREATE INDEX IF NOT EXISTS zerver_realmsession_realm_user_idx | ||
ON zerver_realmsession (realm_id, user_id); | ||
CREATE INDEX IF NOT EXISTS zerver_realmsession_ip_address_idx | ||
ON zerver_realmsession (ip_address); | ||
""" | ||
).format(index_name=Identifier(session_key_index_name)) | ||
) | ||
|
||
|
||
class Migration(migrations.Migration): | ||
atomic = False | ||
|
||
dependencies = [ | ||
("zerver", "0516_sessions_create_custom_model"), | ||
] | ||
|
||
operations = [ | ||
migrations.RunPython(migrate_data_from_django_session_to_realmsession), | ||
] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.