Migration print statements

migrator/management/commands/migrate_old_db.py

@@ -1,6 +1,8 @@
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from django.db import connection
+from django.utils import timezone
+from datetime import datetime, timedelta
 
 from migrator.services.migrate_comments import migrate_comments, migrate_comment_votes
 from migrator.services.migrate_forecasts import (
@@ -29,6 +31,17 @@
 from scoring.models import populate_medal_exclusion_records
 
 
+def print_duration(text, task_start, global_start) -> datetime:
+    print(
+        "\033[92m"
+        f"{text} ---"
+        f"Task Duration:{str(timezone.now() - task_start).split('.')[0]}, "
+        f"Total Runtime:{str(timezone.now() - global_start).split('.')[0]}"
+        "\033[0m"
+    )
+    return timezone.now()
+
+
 class Command(BaseCommand):
     help = """
     Migrates old database data to the new one
@@ -51,6 +64,8 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, site_ids=None, **options):
+        start = timezone.now()
+        task_start = timezone.now()
         site_ids = [int(x) for x in site_ids.split(",")]
         with connection.cursor() as cursor:
             cursor.execute("DROP SCHEMA public CASCADE;")
@@ -60,55 +75,60 @@ def handle(self, *args, site_ids=None, **options):
 
         # main model migration
         migrate_users()
-        print("Migrated users")
+        task_start = print_duration("Migrated users", task_start, start)
         migrate_fab_credits()
+        task_start = print_duration("Migrated fab credits", task_start, start)
         migrate_questions(site_ids=site_ids)
-        print("Migrated questions")
+        task_start = print_duration("Migrated questions", task_start, start)
         migrate_projects(site_ids=site_ids)
-        print("Migrated projects")
+        task_start = print_duration("Migrated projects", task_start, start)
         migrate_votes()
-        print("Migrated votes")
+        task_start = print_duration("Migrated votes", task_start, start)
         migrate_comments()
-        print("Migrated comments")
+        task_start = print_duration("Migrated comments", task_start, start)
         migrate_comment_votes()
-        print("Migrated comment votes")
+        task_start = print_duration("Migrated comment votes", task_start, start)
         migrate_permissions(site_ids=site_ids)
-        print("Migrated permissions")
+        task_start = print_duration("Migrated permissions", task_start, start)
         migrate_forecasts()
-        print("Migrated forecasts")
+        task_start = print_duration("Migrated forecasts", task_start, start)
         migrate_metaculus_predictions()
-        print("Migrated Metaculus predictions")
+        task_start = print_duration("Migrated Metaculus predictions", task_start, start)
         migrate_mailgun_notification_preferences()
-        print("Migrated user notification preferences")
+        task_start = print_duration(
+            "Migrated user notification preferences", task_start, start
+        )
 
         # TODO: enable on prod release!
-        print("\033[93mPost Subscriptions/Following migration is disabled!\033[0m")
+        print(f"\033[93mPost Subscriptions/Following migration is disabled!\033[0m")
         # migrate_subscriptions(site_ids=site_ids)
-        # print("Migrated post subscriptions")
+        # task_start = print_duration("Migrated post subscriptions", task_start, start)
 
         # scoring
         migrate_archived_scores()
-        print("Migrated archived scores")
+        task_start = print_duration("Migrated archived scores", task_start, start)
         score_questions(start_id=options["start_score_questions_with_id"])
-        print("Scored questions")
+        task_start = print_duration("Scored questions", task_start, start)
         populate_medal_exclusion_records()
-        print("Populated medal exclusion records")
+        task_start = print_duration(
+            "Populated medal exclusion records", task_start, start
+        )
         create_global_leaderboards()
-        print("Created global leaderboards")
+        task_start = print_duration("Created global leaderboards", task_start, start)
         populate_global_leaderboards()
-        print("Populated global leaderboards")
+        task_start = print_duration("Populated global leaderboards", task_start, start)
         populate_project_leaderboards()
-        print("Populated project leaderboards")
+        task_start = print_duration("Populated project leaderboards", task_start, start)
 
         # stats on questions
-        print("Running calculate divergence")
         post_migrate_calculate_divergence()
-        print("Running calculate movement")
+        task_start = print_duration("calculated divergence", task_start, start)
         job_compute_movement()
-        print("Running build forecasts")
+        task_start = print_duration("calculated movement", task_start, start)
         call_command("build_forecasts")
-        print("Running compute hotness")
+        task_start = print_duration("built forecasts", task_start, start)
         compute_hotness()
+        task_start = print_duration("computed hotness", task_start, start)
 
         # Reset sql sequences
         reset_sequence()

migrator/services/migrate_comments.py

@@ -2,6 +2,7 @@
 from migrator.utils import paginated_query
 from posts.models import Post
 from questions.models import Forecast
+from django.utils import timezone
 
 
 def create_comment_vote(vote_obj):
@@ -52,31 +53,61 @@ def migrate_comment_votes():
     comment_ids = Comment.objects.values_list("id", flat=True)
     vote_instances = []
 
-    vote_instances += [
-        create_comment_vote(obj)
-        for obj in paginated_query("SELECT * FROM metac_question_comment_likes")
-        if obj["comment_id"] in comment_ids
-    ]
-
+    start = timezone.now()
+    for i, obj in paginated_query("SELECT * FROM metac_question_comment_likes"):
+        print(
+            f"\033[Kmigrating comment votes: {i}. "
+            f"dur:{str(timezone.now() - start).split('.')[0]} ",
+            end="\r",
+        )
+        if obj["comment_id"] in comment_ids:
+            vote_instances.append(create_comment_vote(obj))
+
+    print(
+        f"\033[Kmigrating comment votes: {i}. "
+        f"dur:{str(timezone.now() - start).split('.')[0]} ",
+        "bulk creating...",
+        end="\r",
+    )
     CommentVote.objects.bulk_create(
         vote_instances, ignore_conflicts=True, batch_size=1_000
     )
+    print(
+        f"\033[Kmigrating comment votes: {i}. "
+        f"dur:{str(timezone.now() - start).split('.')[0]} ",
+        "bulk creating... DONE",
+    )
 
 
 def migrate_comments():
     comments = []
     post_ids = Post.objects.values_list("id", flat=True)
 
-    for comment in paginated_query(
-        """
+    start = timezone.now()
+    for i, comment in enumerate(
+        paginated_query(
+            """
         SELECT c.*
         FROM metac_question_comment c
         where c.author_id is not null
         and c.id is not null
         order by c.id
         ;"""
+        ),
+        1,
     ):
+        print(
+            f"\033[Kmigrating comments: {i}. "
+            f"dur:{str(timezone.now() - start).split('.')[0]} ",
+            end="\r",
+        )
         if comment["question_id"] in post_ids:
             comments.append(create_comment(comment))
+    print(
+        f"\033[Kmigrating comments: {i}. "
+        f"dur:{str(timezone.now() - start).split('.')[0]} ",
+    )
 
+    print("bulk creating...", end="\r")
     Comment.objects.bulk_create(comments)
+    print("bulk creating... DONE")

migrator/services/migrate_forecasts.py

@@ -181,7 +181,7 @@ def migrate_metaculus_predictions():
     for i, mp_data in enumerate(mp_histories, 1):
         print(
             f"Migrating continuous metaculus prediction {i}/{c} "
-            f"dur:{str(timezone.now() - start).split(".")[0]} "
+            f"dur:{str(timezone.now() - start).split('.')[0]} "
             f"remaining:{str((timezone.now() - start) / i * (c - i)).split(".")[0]}",
             end="\r",
         )
@@ -244,7 +244,7 @@ def migrate_metaculus_predictions():
         AggregateForecast.objects.bulk_create(forecasts)
     print(
         f"\033[KMigrating continuous metaculus prediction {i}/{c} "
-        f"dur:{str(timezone.now() - start).split(".")[0]} "
+        f"dur:{str(timezone.now() - start).split('.')[0]} "
     )
 
     # binary questions
@@ -262,7 +262,7 @@ def migrate_metaculus_predictions():
     for i, q_data in enumerate(question_histories, 1):
         print(
             f"Migrating binary metaculus prediction {i}/{c} "
-            f"dur:{str(timezone.now() - start).split(".")[0]} "
+            f"dur:{str(timezone.now() - start).split('.')[0]} "
             f"remaining:{str((timezone.now() - start) / i * (c - i)).split(".")[0]}",
             end="\r",
         )
@@ -300,5 +300,5 @@ def migrate_metaculus_predictions():
         AggregateForecast.objects.bulk_create(forecasts)
     print(
         f"\033[KMigrating binary metaculus prediction {i}/{c} "
-        f"dur:{str(timezone.now() - start).split(".")[0]} "
+        f"dur:{str(timezone.now() - start).split('.')[0]} "
     )

migrator/services/migrate_leaderboards.py

@@ -52,15 +52,17 @@ def populate_global_leaderboards():
     for i, leaderboard in enumerate(global_leaderboards, 1):
         print("populating:", i, "/", c, leaderboard.name, end="\r")
         update_project_leaderboard(main_site_project, leaderboard)
+        entries = LeaderboardEntry.objects.filter(leaderboard=leaderboard).count()
         print(
-            "populating:",
+            "\033[Kpopulating:",
             i,
             "/",
             c,
             leaderboard.name,
             "(created",
-            LeaderboardEntry.objects.filter(leaderboard=leaderboard).count(),
+            entries,
             "entries)",
+            end="\r" if entries == 0 else "\n",
         )
 
 
@@ -73,13 +75,15 @@ def populate_project_leaderboards():
         print("populating:", i, "/", c, project.name, end="\r")
         for leaderboard in project.leaderboards.all():
             update_project_leaderboard(project, leaderboard)
+        entries = LeaderboardEntry.objects.filter(leaderboard__project=project).count()
         print(
-            "populating:",
+            "\033[Kpopulating:",
             i,
             "/",
             c,
             project.name,
             "(created",
-            LeaderboardEntry.objects.filter(leaderboard__project=project).count(),
+            entries,
             "entries)",
+            end="\r" if entries == 0 else "\n",
         )

migrator/services/migrate_permissions.py

@@ -138,7 +138,8 @@ def migrate_common_permissions(site_ids: list):
                 ):
                     print(
                         f"QuestionProjectPermission.permission affected "
-                        f"project: {user_project_perm_obj['project_id']}"
+                        f"project: {user_project_perm_obj['project_id']}",
+                        end="\r",
                     )
 
         question_permission = convert_question_permissions(question_permission_code)
@@ -152,9 +153,20 @@ def migrate_common_permissions(site_ids: list):
                 )
             )
 
+    print(
+        f"QuestionProjectPermission.permission affected "
+        f"project: {user_project_perm_obj['project_id']}",
+        "bulk creating...",
+        end="\r",
+    )
     ProjectUserPermission.objects.bulk_create(
         user_project_perms, batch_size=50_000, ignore_conflicts=True
     )
+    print(
+        f"QuestionProjectPermission.permission affected "
+        f"project: {user_project_perm_obj['project_id']}",
+        "bulk creating... DONE",
+    )
     print(
         f"Missed projects: {len(total_missed_project_ids)} "
         f"of the following OLD types: {total_missed_project_types}"

migrator/services/migrate_projects.py

@@ -177,9 +177,6 @@ def migrate_topics(question_ids: list[int], q_p_m2m_cls):
             )
         )
 
-        print("related_category_ids", related_category_ids)
-        print("related_tag_ids", related_tag_ids)
-        print("related_project_ids", related_project_ids)
 
         #
         # Aggregating all M2M tables which were related to the questions
@@ -219,7 +216,13 @@ def migrate_topics(question_ids: list[int], q_p_m2m_cls):
 
         # Some topics contain inline question ids in topic.question_ids column
         m2m_queries.append([{"question_id": x} for x in topic_obj["question_ids"]])
-        print("inline_question_ids", topic_obj["question_ids"])
+        print(
+            f"related_categories:{related_category_ids} "
+            f"related_tags:{related_tag_ids} "
+            f"related_projects:{related_project_ids} "
+            f"inline_questions:{topic_obj["question_ids"]}",
+            end="\r",
+        )
 
         m2m_objects = []
         for m2m in itertools.chain(*m2m_queries):