From d4fc6b1b5836c13205a671974d2a66bec412e6b8 Mon Sep 17 00:00:00 2001 From: gleb Date: Mon, 26 Aug 2024 16:21:39 +0200 Subject: [PATCH 1/2] Remove default Public project from migrator script --- .../management/commands/migrate_old_db.py | 11 ---- migrator/services/migrate_projects.py | 66 +++++++++---------- migrator/services/migrate_questions.py | 13 ++-- 3 files changed, 41 insertions(+), 49 deletions(-) diff --git a/migrator/management/commands/migrate_old_db.py b/migrator/management/commands/migrate_old_db.py index 50ec6c3fde..801db2d0ae 100644 --- a/migrator/management/commands/migrate_old_db.py +++ b/migrator/management/commands/migrate_old_db.py @@ -22,8 +22,6 @@ from migrator.utils import reset_sequence from posts.jobs import job_compute_movement from posts.services.common import compute_hotness -from projects.models import Project -from projects.permissions import ObjectPermission from scoring.models import populate_medal_exclusion_records @@ -49,15 +47,6 @@ def handle(self, *args, site_ids=None, **options): call_command("makemigrations") call_command("migrate") - Project.objects.get_or_create( - type=Project.ProjectTypes.SITE_MAIN, - defaults={ - "name": "Metaculus Community", - "type": Project.ProjectTypes.SITE_MAIN, - "default_permission": ObjectPermission.FORECASTER, - }, - ) - # main model migration migrate_users() print("Migrated users") diff --git a/migrator/services/migrate_projects.py b/migrator/services/migrate_projects.py index 2fec5fbee5..152df52aec 100644 --- a/migrator/services/migrate_projects.py +++ b/migrator/services/migrate_projects.py @@ -29,6 +29,9 @@ def create_project(project_obj: dict) -> Project: "TO": Project.ProjectTypes.TOURNAMENT, "QS": Project.ProjectTypes.QUESTION_SERIES, "MP": Project.ProjectTypes.SITE_MAIN, + # https://www.notion.so/metaculus/Private-Projects-that-aren-t-tournaments-or-question-series-should-become-private-question-series-02cb0bedd6a54650869879b6323d96ff + # Explicitly converting other types to QUESTION_SERIES + "ST": Project.ProjectTypes.QUESTION_SERIES }.get(project_obj["type"]) leaderboard_score_type = None @@ -43,39 +46,36 @@ def create_project(project_obj: dict) -> Project: elif project_obj["score_type"] == "SPOT_PEER_SCORE": leaderboard_score_type = Leaderboard.ScoreTypes.SPOT_PEER_TOURNAMENT - if project_type == Project.ProjectTypes.SITE_MAIN: - project = Project.objects.get(type=project_type) - else: - project = Project( - # We keep original IDS for old projects - id=project_obj["id"], - type=project_type, - name=project_obj["name"], - slug=project_obj["slug"], - subtitle=project_obj["subtitle"], - description=project_obj["description"], - header_image=project_obj["header_image"], - header_logo=project_obj["header_logo"], - prize_pool=project_obj["prize_pool"], - close_date=project_obj["tournament_close_date"], - start_date=project_obj["tournament_start_date"], - sign_up_fields=sign_up_fields, - meta_description=project_obj["meta_description"], - created_at=project_obj["created_at"], - edited_at=project_obj["edited_at"], - # Old project.default_question_permissions was not working - # And project visibility was determined by `is_public` attr - default_permission=( - ObjectPermission.FORECASTER - if (project_obj["public"] and project_obj["id"] != 3349) - else None - ), - ) + project = Project( + # We keep original IDS for old projects + id=project_obj["id"], + type=project_type, + name=project_obj["name"], + slug=project_obj["slug"], + subtitle=project_obj["subtitle"], + description=project_obj["description"], + header_image=project_obj["header_image"], + header_logo=project_obj["header_logo"], + prize_pool=project_obj["prize_pool"], + close_date=project_obj["tournament_close_date"], + start_date=project_obj["tournament_start_date"], + sign_up_fields=sign_up_fields, + meta_description=project_obj["meta_description"], + created_at=project_obj["created_at"], + edited_at=project_obj["edited_at"], + # Old project.default_question_permissions was not working + # And project visibility was determined by `is_public` attr + default_permission=( + ObjectPermission.FORECASTER + if (project_obj["public"] and project_obj["id"] != 3349) + else None + ), + ) - if project_obj["id"] == 3349: - # the FAB project (id == 3349), was probably the only project using the default_question_permissions - # to make the project visible to everyone, but predictable only by the users added to the project - project.default_permission = ObjectPermission.VIEWER + if project_obj["id"] == 3349: + # the FAB project (id == 3349), was probably the only project using the default_question_permissions + # to make the project visible to everyone, but predictable only by the users added to the project + project.default_permission = ObjectPermission.VIEWER project.save() if leaderboard_score_type: @@ -240,7 +240,7 @@ def migrate_projects(site_ids: list[int] = None): for project_obj in paginated_query( "SELECT * FROM metac_project_project " - "WHERE site_id in %s AND (type in ('TO', 'QS') OR (type = 'MP' and site_id = 1))", + "WHERE site_id in %s AND type in ('TO', 'QS', 'MP', 'ST')", [tuple(site_ids)], ): project = create_project(project_obj) diff --git a/migrator/services/migrate_questions.py b/migrator/services/migrate_questions.py index 4beb717241..fb0ad2a78d 100644 --- a/migrator/services/migrate_questions.py +++ b/migrator/services/migrate_questions.py @@ -12,7 +12,6 @@ from posts.models import Notebook, Post, PostUserSnapshot from projects.models import Project from projects.permissions import ObjectPermission -from projects.services import get_site_main_project from questions.constants import ResolutionType from questions.models import Question, Conditional, GroupOfQuestions, Forecast from utils.the_math.formulas import unscaled_location_to_string_location @@ -147,7 +146,6 @@ def create_post(question: dict, **kwargs) -> Post: published_at=question["publish_time"], created_at=question["created_time"], edited_at=question["edited_time"], - default_project=get_site_main_project(), **kwargs, ) @@ -434,7 +432,7 @@ def migrate_questions__notebook(root_questions: list[dict]): notebook.save() post.save() - post.projects.set([get_site_main_project(), *projects]) + post.projects.set(projects) post.save() @@ -477,7 +475,7 @@ def get_children_relation_id_by_attr(id_attr: str): ) except Exception: old_question_yes = None - + try: old_question_no = next( q for q in root_question["children"] if q["qc_resolution"] == 0 @@ -488,7 +486,12 @@ def get_children_relation_id_by_attr(id_attr: str): condition_id = get_children_relation_id_by_attr("condition_id") condition_child_id = get_children_relation_id_by_attr("condition_child_id") - if not condition_id or not condition_child_id or not old_question_yes or not old_question_no: + if ( + not condition_id + or not condition_child_id + or not old_question_yes + or not old_question_no + ): print( f"\n\nError migrating conditionl: {root_question['id']}Could not find all related questions for the conditional pair. old_question_yes: {old_question_yes}, old_question_no: {old_question_no}, condition_id: {condition_id}, condition_child_id: {condition_child_id}" ) From 79ff1759b46f570e38f8effb9d3a621bdf896ea6 Mon Sep 17 00:00:00 2001 From: gleb Date: Mon, 26 Aug 2024 17:17:49 +0200 Subject: [PATCH 2/2] - Converted old project types (e.g. ST) to Question Series - Small fixes --- migrator/management/commands/migrate_old_db.py | 2 +- migrator/services/migrate_permissions.py | 10 +++++----- migrator/services/migrate_projects.py | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/migrator/management/commands/migrate_old_db.py b/migrator/management/commands/migrate_old_db.py index 801db2d0ae..dd693395b3 100644 --- a/migrator/management/commands/migrate_old_db.py +++ b/migrator/management/commands/migrate_old_db.py @@ -62,7 +62,7 @@ def handle(self, *args, site_ids=None, **options): print("Migrated comments") migrate_comment_votes() print("Migrated comment votes") - migrate_permissions() + migrate_permissions(site_ids=site_ids) print("Migrated permissions") migrate_mailgun_notification_preferences() print("Migrated user notification preferences") diff --git a/migrator/services/migrate_permissions.py b/migrator/services/migrate_permissions.py index cb008d1004..964b8a8522 100644 --- a/migrator/services/migrate_permissions.py +++ b/migrator/services/migrate_permissions.py @@ -72,7 +72,7 @@ def convert_project_permissions(code): }.get(code) -def migrate_common_permissions(): +def migrate_common_permissions(site_ids: list): """ Migrates permissions from regular projects """ @@ -101,8 +101,8 @@ def migrate_common_permissions(): FROM metac_project_userprojectpermissions upp JOIN metac_project_project p ON upp.project_id = p.id - WHERE p.type != 'PP' - """ + WHERE p.type != 'PP' AND site_id in %s + """, [tuple(site_ids)] ): # New app merges Project & Categories & Tags etc. # Tournaments & QS & PP were migrated to Project model with the same Ids as the old ones. @@ -329,8 +329,8 @@ def deduplicate_default_project_and_m2m(): PostProject.objects.filter(Exists(subquery)).delete() -def migrate_permissions(): +def migrate_permissions(site_ids: list): migrate_personal_projects() - migrate_common_permissions() + migrate_common_permissions(site_ids) migrate_post_default_project() deduplicate_default_project_and_m2m() diff --git a/migrator/services/migrate_projects.py b/migrator/services/migrate_projects.py index 152df52aec..e228e5e53d 100644 --- a/migrator/services/migrate_projects.py +++ b/migrator/services/migrate_projects.py @@ -29,11 +29,12 @@ def create_project(project_obj: dict) -> Project: "TO": Project.ProjectTypes.TOURNAMENT, "QS": Project.ProjectTypes.QUESTION_SERIES, "MP": Project.ProjectTypes.SITE_MAIN, - # https://www.notion.so/metaculus/Private-Projects-that-aren-t-tournaments-or-question-series-should-become-private-question-series-02cb0bedd6a54650869879b6323d96ff - # Explicitly converting other types to QUESTION_SERIES - "ST": Project.ProjectTypes.QUESTION_SERIES }.get(project_obj["type"]) + # https://www.notion.so/metaculus/Private-Projects-that-aren-t-tournaments-or-question-series-should-become-private-question-series-02cb0bedd6a54650869879b6323d96ff + # Converting other types to QUESTION_SERIES + project_type = project_type or Project.ProjectTypes.QUESTION_SERIES + leaderboard_score_type = None if project_type in [ Project.ProjectTypes.TOURNAMENT, @@ -240,7 +241,7 @@ def migrate_projects(site_ids: list[int] = None): for project_obj in paginated_query( "SELECT * FROM metac_project_project " - "WHERE site_id in %s AND type in ('TO', 'QS', 'MP', 'ST')", + "WHERE site_id in %s", [tuple(site_ids)], ): project = create_project(project_obj)