OpenConceptLab/ocl_issues#1399 | using SQL raw queries in migrations

core/concepts/migrations/0038_auto_20221122_0522.py

@@ -1,30 +1,33 @@
 # Generated by Django 4.1.1 on 2022-11-22 05:22
 
-from django.db import migrations
+from django.db import migrations, connection
 
 
 def populate_concept_descriptions(apps, schema_editor):
+    import time
+    start = time.time()
     """
     1. Populates new concept_descriptions table from localized_texts table using many-to-many concepts_descriptions.
     2. Count of concepts_descriptions (old) and concept_descriptions (new) should be same after this migration.
     """
-    Concept = apps.get_model('concepts', 'Concept')
-    LocalizedText = apps.get_model('concepts', 'LocalizedText')
-    ConceptDescription = apps.get_model('concepts', 'ConceptDescription')
-    descriptions = []
-    for concept_description in Concept.descriptions.through.objects.iterator(chunk_size=1000):
-        localized_text = LocalizedText.objects.filter(id=concept_description.localizedtext_id).first()
-        if localized_text:
-            descriptions.append(ConceptDescription(
-                concept_id=concept_description.concept_id,
-                external_id=localized_text.external_id,
-                name=localized_text.name,
-                type=localized_text.type,
-                locale=localized_text.locale,
-                locale_preferred=localized_text.locale_preferred,
-                created_at=localized_text.created_at,
-            ))
-    ConceptDescription.objects.bulk_create(descriptions, batch_size=1000)
+    with connection.cursor() as cursor:
+        cursor.execute("""select concept_id, localizedtext_id from concepts_descriptions;""")
+        rows = cursor.fetchall()
+
+        statement = "insert into concept_descriptions(concept_id, external_id, name, type, locale, locale_preferred, created_at) values "
+        for row in rows:
+            cursor.execute(
+                "select external_id, name, type, locale, locale_preferred, created_at from localized_texts where id = %s",
+                [row[1]]
+            )
+            locale = cursor.fetchone()
+            name = locale[1].replace("'", "''")
+            statement += f"({row[0]}, '{locale[0]}', '{name}', '{locale[2]}', '{locale[3]}', {locale[4]}, '{locale[5]}'),"
+        if statement.endswith(','):
+            statement = statement[:-1] + ';'
+        cursor.execute(statement)
+
+    print('Time: ', time.time() - start)
 
 
 class Migration(migrations.Migration):

core/concepts/migrations/0039_auto_20221122_0552.py

@@ -1,9 +1,12 @@
 # Generated by Django 4.1.1 on 2022-11-22 05:52
 
-from django.db import migrations
+from django.db import migrations, connection
+from pydash import flatten
 
 
 def add_names_for_latest_version(apps, schema_editor):
+    import time
+    start = time.time()
     """
     1. Populates LocalizedText with copy of names of Concept for its latest version.
     2. Concept and its latest version used to share names (and descriptions) using many-to-many (concepts_names).
@@ -14,28 +17,28 @@ def add_names_for_latest_version(apps, schema_editor):
        - concepts_names for latest versions needs to be deleted for the next migration to populate concept_id in
          localized_texts. Latest versions and Concept itself uses the same localizedtext_id.
     """
-    Concept = apps.get_model('concepts', 'Concept')
-    LocalizedText = apps.get_model('concepts', 'LocalizedText')
-    names = []
-    concepts_names_to_delete = []
-    for latest_version in Concept.objects.filter(is_latest_version=True).iterator(chunk_size=1000):
-        for concept_name in Concept.names.through.objects.filter(concept_id=latest_version.id):
-            concepts_names_to_delete.append(concept_name.id)
-            localized_text = LocalizedText.objects.filter(id=concept_name.localizedtext_id).first()
-            if localized_text:
-                concepts_names_to_delete.append(concept_name.id)
-                names.append(LocalizedText(
-                    concept_id=latest_version.id,
-                    external_id=localized_text.external_id,
-                    name=localized_text.name,
-                    type=localized_text.type,
-                    locale=localized_text.locale,
-                    locale_preferred=localized_text.locale_preferred,
-                    created_at=localized_text.created_at,
-                ))
-
-    LocalizedText.objects.bulk_create(names, batch_size=1000)
-    Concept.names.through.objects.filter(id__in=concepts_names_to_delete).delete()
+    with connection.cursor() as cursor:
+        cursor.execute("select id from concepts where is_latest_version=true;")
+        statement = "insert into localized_texts(concept_id, external_id, name, type, locale, locale_preferred, created_at) values "
+        concepts = cursor.fetchall()
+        for concept in concepts:
+            cursor.execute("select localizedtext_id from concepts_names where concept_id = %s", concept)
+            locales = cursor.fetchall()
+            for locale in locales:
+                cursor.execute(
+                    "select external_id, name, type, locale, locale_preferred, created_at from localized_texts where id = %s",
+                    [locale]
+                )
+                _locale = cursor.fetchone()
+                name = _locale[1].replace("'", "''")
+                statement += f"({concept[0]}, '{_locale[0]}', '{name}', '{_locale[2]}', '{_locale[3]}', {_locale[4]}, '{_locale[5]}'),"
+        if statement.endswith(','):
+            statement = statement[:-1] + ';'
+
+        cursor.execute(statement)
+        cursor.execute(f'delete from concepts_names where concept_id in {tuple(flatten(concepts))}')
+
+    print('Time: ', time.time() - start)
 
 
 class Migration(migrations.Migration):

core/concepts/migrations/0040_auto_20221122_0613.py

@@ -1,25 +1,31 @@
 # Generated by Django 4.1.1 on 2022-11-22 06:13
 
-from django.db import migrations
+from django.db import migrations, connection
 
 
 def populate_concept_id_in_localized_texts(apps, schema_editor):
+    import time
+    start = time.time()
     """
     1. Populates concept_id in localized_texts table using concepts_names (many-to-many).
     2. After this migration:
        - localized_texts table will have concept_id for each name
        - localized_texts without any concept_id can be deleted, as they are all descriptions which were migrated earlier
     """
-    Concept = apps.get_model('concepts', 'Concept')
-    LocalizedText = apps.get_model('concepts', 'LocalizedText')
-    names = []
-    for concept_name in Concept.names.through.objects.iterator(chunk_size=1000):
-        localized_text = LocalizedText.objects.filter(id=concept_name.localizedtext_id).first()
-        if localized_text:
-            localized_text.concept_id = concept_name.concept_id
-            names.append(localized_text)
-
-    LocalizedText.objects.bulk_update(names, fields=['concept_id'], batch_size=1000)
+    with connection.cursor() as cursor:
+        cursor.execute("select concept_id, localizedtext_id from concepts_names;")
+        names = cursor.fetchall()
+        statement = f"update localized_texts as l set concept_id = n.concept_id from (values"
+        for name in names:
+            statement += f'{name},'
+        if statement.endswith(','):
+            statement = statement[:-1]
+
+        statement += ') as n(concept_id, localizedtext_id) where n.localizedtext_id = l.id;'
+
+        cursor.execute(statement)
+
+    print('Time: ', time.time() - start)
 
 
 class Migration(migrations.Migration):