From 330d5023dffa664334d0a71ed99888342dd321bf Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Mon, 10 Sep 2018 19:56:24 +0300 Subject: [PATCH 1/6] write migrations --- .gitignore | 1 + ckan/migration/README | 5 +- ckan/migration/alembic.ini | 74 ++++++ ckan/migration/env.py | 73 ++++++ ckan/migration/manage.py | 6 - ckan/migration/migrate.cfg | 20 -- ckan/migration/script.py.mako | 24 ++ ...001_103676e0a497_create_existing_tables.py | 154 ++++++++++++ .../versions/001_add_existing_tables.py | 115 --------- ..._86fdd8c54775_add_author_and_maintainer.py | 34 +++ .../versions/002_add_author_and_maintainer.py | 34 --- .../migration/versions/003_add_user_object.py | 27 --- .../003_f22b4f5241a5_add_user_object.py | 34 +++ .../versions/004_add_group_object.py | 38 --- .../004_f92ee205e46d_add_group_object.py | 40 ++++ ...5_12c2232c15f5_add_authorization_tables.py | 61 +++++ .../versions/005_add_authorization_tables.py | 57 ----- ckan/migration/versions/006_add_ratings.py | 30 --- .../versions/006_c83955e7acb6_add_ratings.py | 31 +++ .../007_1928d4af1cda_add_system_roles.py | 31 +++ .../versions/007_add_system_roles.py | 27 --- .../008_e8283ffb257e_update_vdm_ids.py | 53 +++++ ckan/migration/versions/008_update_vdm_ids.py | 91 -------- .../versions/009_add_creation_timestamps.py | 27 --- ...09_b739a48de5c4_add_creation_timestamps.py | 27 +++ .../010_a6f13bf14d0c_add_user_about.py | 23 ++ ckan/migration/versions/010_add_user_about.py | 16 -- ..._866f6370b4ac_add_package_search_vector.py | 31 +++ .../versions/011_add_package_search_vector.py | 25 -- ckan/migration/versions/012_add_resources.py | 80 ------- .../012_e5ca33a5d445_add_resources.py | 63 +++++ .../versions/013_8a3a5af39797_add_hash.py | 23 ++ ckan/migration/versions/013_add_hash.py | 17 -- .../versions/014_93519b684820_hash_2.py | 25 ++ ckan/migration/versions/014_hash_2.py | 19 -- .../015_6d8ffebcaf54_remove_state_object.py | 41 ++++ .../versions/015_remove_state_object.py | 35 --- .../016_37ada738328e_uuids_everywhere.py | 127 ++++++++++ .../versions/016_uuids_everywhere.py | 196 ---------------- .../017_1250b2ff3e36_add_pkg_relationships.py | 58 +++++ .../versions/017_add_pkg_relationships.py | 41 ---- .../018_05a0778051ca_adjust_licenses.py | 40 ++++ .../migration/versions/018_adjust_licenses.py | 180 -------------- ...19_b2eb6f34a638_pkg_relationships_state.py | 27 +++ .../versions/019_pkg_relationships_state.py | 26 --- .../020_69a0b0efc609_add_changeset.py | 68 ++++++ ckan/migration/versions/020_add_changeset.py | 47 ---- ...021_765143af2ba3_postgresql_upgrade_sql.py | 102 ++++++++ .../versions/021_postgres_downgrade.sql | 1 - .../versions/021_postgres_upgrade.sql | 73 ------ .../versions/021_postgresql_downgrade.sql | 1 - .../versions/021_postgresql_upgrade.sql | 73 ------ .../022_7b324ca6c0dc_add_group_extras.py | 29 +++ .../versions/022_add_group_extras.py | 53 ----- .../023_87fdd05f0744_add_harvesting.py | 49 ++++ ckan/migration/versions/023_add_harvesting.py | 42 ---- ...024_12981fe12484_add_harvested_document.py | 29 +++ .../versions/024_add_harvested_document.py | 22 -- .../versions/025_add_authorization_groups.py | 58 ----- ...5_b581622ad327_add_authorization_groups.py | 76 ++++++ ...615b25af443_authorization_group_user_pk.py | 23 ++ .../026_authorization_group_user_pk.py | 37 --- .../027_11e5745c6fc9_adjust_harvester.py | 39 ++++ .../versions/027_adjust_harvester.py | 34 --- ...cdd68fe9ba21_drop_harvest_source_status.py | 23 ++ .../028_drop_harvest_source_status.py | 18 -- .../029_1bfdf4240915_version_groups.py | 106 +++++++++ ckan/migration/versions/029_version_groups.py | 174 -------------- .../030_additional_user_attributes.py | 24 -- ...b16cbf164c8a_additional_user_attributes.py | 29 +++ ...1_1b05245167d6_move_openid_to_new_field.py | 23 ++ .../versions/031_move_openid_to_new_field.py | 39 ---- .../032_add_extra_info_field_to_resources.py | 20 -- ...1422d_add_extra_info_field_to_resources.py | 27 +++ ...df15_auth_group_user_id_add_conditional.py | 26 +++ .../033_auth_group_user_id_add_conditional.py | 32 --- .../034_6c600693af5b_resource_group_table.py | 158 +++++++++++++ .../versions/034_resource_group_table.py | 209 ----------------- ..._81148ccebd6c_harvesting_doc_versioning.py | 91 ++++++++ .../versions/035_harvesting_doc_versioning.py | 50 ---- .../036_ecaa8b38782f_lockdown_roles.py | 23 ++ ckan/migration/versions/036_lockdown_roles.py | 37 --- .../037_edcf3b8c3c1b_role_anon_editor.py | 23 ++ .../versions/037_role_anon_editor.py | 50 ---- .../versions/038_delete_migration_tables.py | 12 - ...38_fd6622e3d964_delete_migration_tables.py | 101 ++++++++ .../versions/039_add_expired_id_and_dates.py | 221 ------------------ ...9_cca459c76d45_add_expired_id_and_dates.py | 156 +++++++++++++ .../040_500a08f4818e_reset_key_on_user.py | 23 ++ .../versions/040_reset_key_on_user.py | 15 -- .../041_6817d4e3bdc3_resource_new_fields.py | 43 ++++ .../versions/041_resource_new_fields.py | 34 --- .../042_da65e2877034_user_revision_indexes.py | 33 +++ .../versions/042_user_revision_indexes.py | 15 -- .../043_bd38cd6502b2_drop_postgres_search.py | 31 +++ .../versions/043_drop_postgres_search.py | 14 -- .../044_4190eeeb8d73_add_task_status.py | 41 ++++ .../migration/versions/044_add_task_status.py | 26 --- .../045_54e3f155d945_user_name_unique.py | 23 ++ .../versions/045_user_name_unique.py | 19 -- .../046_b69e9b80396f_drop_changesets.py | 68 ++++++ .../migration/versions/046_drop_changesets.py | 12 - ...83a7c406926_rename_package_group_member.py | 168 +++++++++++++ .../047_rename_package_group_member.py | 110 --------- ...a7011172b3f_add_activity_streams_tables.py | 50 ++++ .../048_add_activity_streams_tables.py | 37 --- .../versions/049_add_group_approval_status.py | 19 -- ..._e0c06c2177b5_add_group_approval_status.py | 27 +++ ...050_01a6b058cb7f_term_translation_table.py | 29 +++ .../versions/050_term_translation_table.py | 22 -- .../051_a4fb0d85ced6_add_tag_vocabulary.py | 43 ++++ .../versions/051_add_tag_vocabulary.py | 31 --- ...2_ba693d64c6d7_update_member_capacities.py | 23 ++ .../versions/052_update_member_capacities.py | 14 -- .../053_9d051a099097_add_group_logo.py | 25 ++ ckan/migration/versions/053_add_group_logo.py | 14 -- .../versions/054_add_resource_created_date.py | 11 - ..._da21b38da4db_add_resource_created_date.py | 25 ++ ...db947bf_update_user_and_activity_detail.py | 25 ++ .../055_update_user_and_activity_detail.py | 11 - .../056_11af3215ae89_add_related_table.py | 49 ++++ .../versions/056_add_related_table.py | 42 ---- .../versions/057_660a5aae527e_tracking.py | 59 +++++ ckan/migration/versions/057_tracking.py | 35 --- .../versions/058_add_follower_tables.py | 39 ---- .../058_bd36d1826a5d_add_follower_tables.py | 73 ++++++ ...9291bb46f352_add_related_count_and_flag.py | 33 +++ .../059_add_related_count_and_flag.py | 16 -- .../060_31ad11c518fc_add_system_info_table.py | 47 ++++ .../versions/060_add_system_info_table.py | 29 --- ...1_338d460bc460_add_follower_group_table.py | 49 ++++ .../versions/061_add_follower__group_table.py | 24 -- .../062_6deb2bbab394_add_dashboard_table.py | 34 +++ .../versions/062_add_dashboard_table.py | 18 -- .../versions/063_8b633852cb7a_org_changes.py | 54 +++++ ckan/migration/versions/063_org_changes.py | 33 --- ...4f8becd4919a_add_email_last_sent_column.py | 31 +++ .../064_add_email_last_sent_column.py | 12 - ...439e_add_email_notifications_preference.py | 30 +++ .../065_add_email_notifications_preference.py | 12 - .../066_ad16b3bd8cb6_default_package_type.py | 23 ++ .../versions/066_default_package_type.py | 14 -- ...067_266c110eafec_turn_extras_to_strings.py | 23 ++ .../versions/067_turn_extras_to_strings.py | 30 --- .../versions/068_add_package_extras_index.py | 10 - ...8_e33a5f2b2a84_add_package_extras_index.py | 26 +++ ...5cdb_resource_url_and_metadata_modified.py | 41 ++++ .../069_resource_url_and_metadata_modified.py | 76 ------ .../070_add_activity_and_resource_indexes.py | 16 -- ...12fa7_add_activity_and_resource_indexes.py | 38 +++ .../071_add_state_column_to_user_table.py | 11 - ...081ef73a_add_state_column_to_user_table.py | 28 +++ .../072_08dcb9233ad7_add_resource_view.py | 38 +++ .../versions/072_add_resource_view.py | 26 --- ...08be3_update_resource_view_resource_id_.py | 34 +++ ...te_resource_view_resource_id_constraint.py | 17 -- ...074_a4ca55f0f45e_remove_resource_groups.py | 128 ++++++++++ .../versions/074_remove_resource_groups.py | 31 --- .../075_9cdc88c8896a_rename_view_plugins.py | 23 ++ .../versions/075_rename_view_plugins.py | 17 -- .../076_59995aa965c0_rename_view_plugins_2.py | 23 ++ .../versions/076_rename_view_plugins_2.py | 17 -- ...171a04d86d_add_revisions_to_system_info.py | 57 +++++ .../077_add_revisions_to_system_info.py | 25 -- ...078_ae821876532a_remove_old_authz_model.py | 109 +++++++++ .../versions/078_remove_old_authz_model.py | 16 -- ...79_e0177a15d2c9_resource_revision_index.py | 25 ++ .../versions/079_resource_revision_index.py | 10 - .../080_8224d872c64f_continuity_id_indexes.py | 39 ++++ .../versions/080_continuity_id_indexes.py | 16 -- .../081_a64cf4a79182_set_datastore_active.py | 24 ++ .../versions/081_set_datastore_active.py | 67 ------ ...a886d0ede4_create_index_creator_user_id.py | 25 ++ .../082_create_index_creator_user_id.py | 10 - .../083_f98d8fa2a7f7_remove_related_items.py | 72 ++++++ .../versions/083_remove_related_items.py | 28 --- .../versions/084_add_metadata_created.py | 17 -- .../084_d85ce5783688_add_metadata_created.py | 38 +++ .../085_adjust_activity_timestamps.py | 23 -- ...f9bf3d5c4b4d_adjust_activity_timestamps.py | 44 ++++ .../086_19663581b3bb_drop_openid_column.py | 24 ++ .../versions/086_drop_openid_column.py | 12 - ...03cab77_remove_old_authorization_tables.py | 47 ++++ .../087_remove_old_authorization_tables.py | 10 - ckan/migration/versions/__init__.py | 0 requirements.txt | 1 + 186 files changed, 4363 insertions(+), 3601 deletions(-) create mode 100644 ckan/migration/alembic.ini create mode 100644 ckan/migration/env.py delete mode 100644 ckan/migration/manage.py delete mode 100644 ckan/migration/migrate.cfg create mode 100644 ckan/migration/script.py.mako create mode 100644 ckan/migration/versions/001_103676e0a497_create_existing_tables.py delete mode 100644 ckan/migration/versions/001_add_existing_tables.py create mode 100644 ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py delete mode 100644 ckan/migration/versions/002_add_author_and_maintainer.py delete mode 100644 ckan/migration/versions/003_add_user_object.py create mode 100644 ckan/migration/versions/003_f22b4f5241a5_add_user_object.py delete mode 100644 ckan/migration/versions/004_add_group_object.py create mode 100644 ckan/migration/versions/004_f92ee205e46d_add_group_object.py create mode 100644 ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py delete mode 100644 ckan/migration/versions/005_add_authorization_tables.py delete mode 100644 ckan/migration/versions/006_add_ratings.py create mode 100644 ckan/migration/versions/006_c83955e7acb6_add_ratings.py create mode 100644 ckan/migration/versions/007_1928d4af1cda_add_system_roles.py delete mode 100644 ckan/migration/versions/007_add_system_roles.py create mode 100644 ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py delete mode 100644 ckan/migration/versions/008_update_vdm_ids.py delete mode 100644 ckan/migration/versions/009_add_creation_timestamps.py create mode 100644 ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py create mode 100644 ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py delete mode 100644 ckan/migration/versions/010_add_user_about.py create mode 100644 ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py delete mode 100644 ckan/migration/versions/011_add_package_search_vector.py delete mode 100644 ckan/migration/versions/012_add_resources.py create mode 100644 ckan/migration/versions/012_e5ca33a5d445_add_resources.py create mode 100644 ckan/migration/versions/013_8a3a5af39797_add_hash.py delete mode 100644 ckan/migration/versions/013_add_hash.py create mode 100644 ckan/migration/versions/014_93519b684820_hash_2.py delete mode 100644 ckan/migration/versions/014_hash_2.py create mode 100644 ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py delete mode 100644 ckan/migration/versions/015_remove_state_object.py create mode 100644 ckan/migration/versions/016_37ada738328e_uuids_everywhere.py delete mode 100644 ckan/migration/versions/016_uuids_everywhere.py create mode 100644 ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py delete mode 100644 ckan/migration/versions/017_add_pkg_relationships.py create mode 100644 ckan/migration/versions/018_05a0778051ca_adjust_licenses.py delete mode 100644 ckan/migration/versions/018_adjust_licenses.py create mode 100644 ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py delete mode 100644 ckan/migration/versions/019_pkg_relationships_state.py create mode 100644 ckan/migration/versions/020_69a0b0efc609_add_changeset.py delete mode 100644 ckan/migration/versions/020_add_changeset.py create mode 100644 ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py delete mode 100644 ckan/migration/versions/021_postgres_downgrade.sql delete mode 100644 ckan/migration/versions/021_postgres_upgrade.sql delete mode 100644 ckan/migration/versions/021_postgresql_downgrade.sql delete mode 100644 ckan/migration/versions/021_postgresql_upgrade.sql create mode 100644 ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py delete mode 100644 ckan/migration/versions/022_add_group_extras.py create mode 100644 ckan/migration/versions/023_87fdd05f0744_add_harvesting.py delete mode 100644 ckan/migration/versions/023_add_harvesting.py create mode 100644 ckan/migration/versions/024_12981fe12484_add_harvested_document.py delete mode 100644 ckan/migration/versions/024_add_harvested_document.py delete mode 100644 ckan/migration/versions/025_add_authorization_groups.py create mode 100644 ckan/migration/versions/025_b581622ad327_add_authorization_groups.py create mode 100644 ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py delete mode 100644 ckan/migration/versions/026_authorization_group_user_pk.py create mode 100644 ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py delete mode 100644 ckan/migration/versions/027_adjust_harvester.py create mode 100644 ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py delete mode 100644 ckan/migration/versions/028_drop_harvest_source_status.py create mode 100644 ckan/migration/versions/029_1bfdf4240915_version_groups.py delete mode 100644 ckan/migration/versions/029_version_groups.py delete mode 100644 ckan/migration/versions/030_additional_user_attributes.py create mode 100644 ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py create mode 100644 ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py delete mode 100644 ckan/migration/versions/031_move_openid_to_new_field.py delete mode 100644 ckan/migration/versions/032_add_extra_info_field_to_resources.py create mode 100644 ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py create mode 100644 ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py delete mode 100644 ckan/migration/versions/033_auth_group_user_id_add_conditional.py create mode 100644 ckan/migration/versions/034_6c600693af5b_resource_group_table.py delete mode 100644 ckan/migration/versions/034_resource_group_table.py create mode 100644 ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py delete mode 100644 ckan/migration/versions/035_harvesting_doc_versioning.py create mode 100644 ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py delete mode 100644 ckan/migration/versions/036_lockdown_roles.py create mode 100644 ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py delete mode 100644 ckan/migration/versions/037_role_anon_editor.py delete mode 100644 ckan/migration/versions/038_delete_migration_tables.py create mode 100644 ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py delete mode 100644 ckan/migration/versions/039_add_expired_id_and_dates.py create mode 100644 ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py create mode 100644 ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py delete mode 100644 ckan/migration/versions/040_reset_key_on_user.py create mode 100644 ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py delete mode 100644 ckan/migration/versions/041_resource_new_fields.py create mode 100644 ckan/migration/versions/042_da65e2877034_user_revision_indexes.py delete mode 100644 ckan/migration/versions/042_user_revision_indexes.py create mode 100644 ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py delete mode 100644 ckan/migration/versions/043_drop_postgres_search.py create mode 100644 ckan/migration/versions/044_4190eeeb8d73_add_task_status.py delete mode 100644 ckan/migration/versions/044_add_task_status.py create mode 100644 ckan/migration/versions/045_54e3f155d945_user_name_unique.py delete mode 100644 ckan/migration/versions/045_user_name_unique.py create mode 100644 ckan/migration/versions/046_b69e9b80396f_drop_changesets.py delete mode 100644 ckan/migration/versions/046_drop_changesets.py create mode 100644 ckan/migration/versions/047_883a7c406926_rename_package_group_member.py delete mode 100644 ckan/migration/versions/047_rename_package_group_member.py create mode 100644 ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py delete mode 100644 ckan/migration/versions/048_add_activity_streams_tables.py delete mode 100644 ckan/migration/versions/049_add_group_approval_status.py create mode 100644 ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py create mode 100644 ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py delete mode 100644 ckan/migration/versions/050_term_translation_table.py create mode 100644 ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py delete mode 100644 ckan/migration/versions/051_add_tag_vocabulary.py create mode 100644 ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py delete mode 100644 ckan/migration/versions/052_update_member_capacities.py create mode 100644 ckan/migration/versions/053_9d051a099097_add_group_logo.py delete mode 100644 ckan/migration/versions/053_add_group_logo.py delete mode 100644 ckan/migration/versions/054_add_resource_created_date.py create mode 100644 ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py create mode 100644 ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py delete mode 100644 ckan/migration/versions/055_update_user_and_activity_detail.py create mode 100644 ckan/migration/versions/056_11af3215ae89_add_related_table.py delete mode 100644 ckan/migration/versions/056_add_related_table.py create mode 100644 ckan/migration/versions/057_660a5aae527e_tracking.py delete mode 100644 ckan/migration/versions/057_tracking.py delete mode 100644 ckan/migration/versions/058_add_follower_tables.py create mode 100644 ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py create mode 100644 ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py delete mode 100644 ckan/migration/versions/059_add_related_count_and_flag.py create mode 100644 ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py delete mode 100644 ckan/migration/versions/060_add_system_info_table.py create mode 100644 ckan/migration/versions/061_338d460bc460_add_follower_group_table.py delete mode 100644 ckan/migration/versions/061_add_follower__group_table.py create mode 100644 ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py delete mode 100644 ckan/migration/versions/062_add_dashboard_table.py create mode 100644 ckan/migration/versions/063_8b633852cb7a_org_changes.py delete mode 100644 ckan/migration/versions/063_org_changes.py create mode 100644 ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py delete mode 100644 ckan/migration/versions/064_add_email_last_sent_column.py create mode 100644 ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py delete mode 100644 ckan/migration/versions/065_add_email_notifications_preference.py create mode 100644 ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py delete mode 100644 ckan/migration/versions/066_default_package_type.py create mode 100644 ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py delete mode 100644 ckan/migration/versions/067_turn_extras_to_strings.py delete mode 100644 ckan/migration/versions/068_add_package_extras_index.py create mode 100644 ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py create mode 100644 ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py delete mode 100644 ckan/migration/versions/069_resource_url_and_metadata_modified.py delete mode 100644 ckan/migration/versions/070_add_activity_and_resource_indexes.py create mode 100644 ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py delete mode 100644 ckan/migration/versions/071_add_state_column_to_user_table.py create mode 100644 ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py create mode 100644 ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py delete mode 100644 ckan/migration/versions/072_add_resource_view.py create mode 100644 ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py delete mode 100644 ckan/migration/versions/073_update_resource_view_resource_id_constraint.py create mode 100644 ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py delete mode 100644 ckan/migration/versions/074_remove_resource_groups.py create mode 100644 ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py delete mode 100644 ckan/migration/versions/075_rename_view_plugins.py create mode 100644 ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py delete mode 100644 ckan/migration/versions/076_rename_view_plugins_2.py create mode 100644 ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py delete mode 100644 ckan/migration/versions/077_add_revisions_to_system_info.py create mode 100644 ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py delete mode 100644 ckan/migration/versions/078_remove_old_authz_model.py create mode 100644 ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py delete mode 100644 ckan/migration/versions/079_resource_revision_index.py create mode 100644 ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py delete mode 100644 ckan/migration/versions/080_continuity_id_indexes.py create mode 100644 ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py delete mode 100644 ckan/migration/versions/081_set_datastore_active.py create mode 100644 ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py delete mode 100644 ckan/migration/versions/082_create_index_creator_user_id.py create mode 100644 ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py delete mode 100644 ckan/migration/versions/083_remove_related_items.py delete mode 100644 ckan/migration/versions/084_add_metadata_created.py create mode 100644 ckan/migration/versions/084_d85ce5783688_add_metadata_created.py delete mode 100644 ckan/migration/versions/085_adjust_activity_timestamps.py create mode 100644 ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py create mode 100644 ckan/migration/versions/086_19663581b3bb_drop_openid_column.py delete mode 100644 ckan/migration/versions/086_drop_openid_column.py create mode 100644 ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py delete mode 100644 ckan/migration/versions/087_remove_old_authorization_tables.py delete mode 100644 ckan/migration/versions/__init__.py diff --git a/.gitignore b/.gitignore index 39267c75f22..ea6f07afd1b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ tmp/* solr_runtime/* fl_notes.txt *.ini +!ckan/migration/alembic.ini .noseids *~ .idea diff --git a/ckan/migration/README b/ckan/migration/README index 6218f8cac42..98e4f9c44ef 100644 --- a/ckan/migration/README +++ b/ckan/migration/README @@ -1,4 +1 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ +Generic single-database configuration. \ No newline at end of file diff --git a/ckan/migration/alembic.ini b/ckan/migration/alembic.ini new file mode 100644 index 00000000000..eb2da491029 --- /dev/null +++ b/ckan/migration/alembic.ini @@ -0,0 +1,74 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = . + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# timezone to use when rendering the date +# within the migration file as well as the filename. +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to ckan/migration/alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat ckan/migration/alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://ckan_default:root@localhost/ckan_alembic + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ckan/migration/env.py b/ckan/migration/env.py new file mode 100644 index 00000000000..1b451f47470 --- /dev/null +++ b/ckan/migration/env.py @@ -0,0 +1,73 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from ckan.model import init_model +from ckan.model.meta import metadata +target_metadata = metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + with connectable.connect() as connection: + init_model(connection) + + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ckan/migration/manage.py b/ckan/migration/manage.py deleted file mode 100644 index 85da87c325b..00000000000 --- a/ckan/migration/manage.py +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 - -from migrate.versioning.shell import main - -main(repository='ckan/migration') diff --git a/ckan/migration/migrate.cfg b/ckan/migration/migrate.cfg deleted file mode 100644 index bb7c61566b9..00000000000 --- a/ckan/migration/migrate.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=Ckan - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] diff --git a/ckan/migration/script.py.mako b/ckan/migration/script.py.mako new file mode 100644 index 00000000000..2c0156303a8 --- /dev/null +++ b/ckan/migration/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ckan/migration/versions/001_103676e0a497_create_existing_tables.py b/ckan/migration/versions/001_103676e0a497_create_existing_tables.py new file mode 100644 index 00000000000..45b5ed93ee0 --- /dev/null +++ b/ckan/migration/versions/001_103676e0a497_create_existing_tables.py @@ -0,0 +1,154 @@ +"""Create existing tables + +Revision ID: 103676e0a497 +Revises: +Create Date: 2018-09-04 16:57:42.622504 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '103676e0a497' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + + op.create_table( + 'state', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100)), + ) + + op.create_table( + 'revision', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('timestamp', sa.DateTime(timezone=False)), + sa.Column('author', sa.Unicode(200)), + sa.Column('message', sa.UnicodeText()), + sa.Column('state_id', sa.Integer), + ) + + op.create_table( + 'apikey', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.UnicodeText()), + sa.Column('key', sa.UnicodeText()), + ) + + op.create_table( + 'license', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100)), sa.Column('state_id', sa.Integer) + ) + + op.create_table( + 'package', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100), nullable=False, unique=True), + sa.Column('title', sa.UnicodeText()), + sa.Column('version', sa.Unicode(100)), + sa.Column('url', sa.UnicodeText()), + sa.Column('download_url', sa.UnicodeText()), + sa.Column('notes', sa.UnicodeText()), + sa.Column('license_id', sa.Integer, sa.ForeignKey('license.id')), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column('revision_id', sa.Integer, sa.ForeignKey('revision.id')), + ) + + op.create_table( + 'package_revision', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100), nullable=False), + sa.Column('title', sa.UnicodeText()), + sa.Column('version', sa.Unicode(100)), + sa.Column('url', sa.UnicodeText()), + sa.Column('download_url', sa.UnicodeText()), + sa.Column('notes', sa.UnicodeText()), + sa.Column('license_id', sa.Integer, sa.ForeignKey('license.id')), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column( + 'revision_id', + sa.Integer, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column('continuity_id', sa.Integer, sa.ForeignKey('package.id')), + ) + + op.create_table( + 'tag', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100), nullable=False, unique=True), + ) + + op.create_table( + 'package_tag', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id')), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column('revision_id', sa.Integer, sa.ForeignKey('revision.id')), + ) + + op.create_table( + 'package_tag_revision', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('tag_id', sa.Integer, sa.ForeignKey('tag.id')), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column( + 'revision_id', + sa.Integer, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.Integer, sa.ForeignKey('package_tag.id') + ), + ) + + op.create_table( + 'package_extra', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('key', sa.UnicodeText()), + sa.Column('value', sa.UnicodeText()), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column('revision_id', sa.Integer, sa.ForeignKey('revision.id')), + ) + + op.create_table( + 'package_extra_revision', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('key', sa.UnicodeText()), + sa.Column('value', sa.UnicodeText()), + sa.Column('state_id', sa.Integer, sa.ForeignKey('state.id')), + sa.Column( + 'revision_id', + sa.Integer, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.Integer, sa.ForeignKey('package_extra.id') + ), + ) + + +def downgrade(): + op.drop_table('package_extra_revision') + op.drop_table('package_extra') + op.drop_table('package_tag_revision') + op.drop_table('package_tag') + op.drop_table('tag') + op.drop_table('package_revision') + op.drop_table('package') + op.drop_table('license') + op.drop_table('apikey') + op.drop_table('revision') + op.drop_table('state') diff --git a/ckan/migration/versions/001_add_existing_tables.py b/ckan/migration/versions/001_add_existing_tables.py deleted file mode 100644 index c346a889b45..00000000000 --- a/ckan/migration/versions/001_add_existing_tables.py +++ /dev/null @@ -1,115 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -from ckan.common import config as ckan_config - - -def upgrade(migrate_engine): - schema = ckan_config.get(u'ckan.migrations.target_schema') or 'public' - # we specify the schema here because of a clash with another 'state' table - # in the mdillon/postgis container. You only need to change the value in the - # config if you've altered the default schema from 'public' in your - # postgresql.conf. Because this is such a rarely needed option, it is - # otherwise undocumented. - meta = MetaData(schema=schema) - - state = Table('state', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('name', Unicode(100)), - ) - - revision = Table('revision', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('timestamp', DateTime(timezone=False) ), - Column('author', Unicode(200)), - Column('message', UnicodeText()), - Column('state_id', Integer() ), - ) - - apikey = Table('apikey', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('name', UnicodeText()), - Column('key', UnicodeText()), - ) - - license = Table('license', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('name', Unicode(100)), - # Column('state_id', Integer(), ForeignKey('state.id')), - Column('state_id', Integer()) - ) - - package = Table('package', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('name', Unicode(100) , nullable=False, unique=True), - Column('title', UnicodeText()), - Column('version', Unicode(100)), - Column('url', UnicodeText()), - Column('download_url', UnicodeText()), - Column('notes', UnicodeText()), - Column('license_id', Integer(), ForeignKey('license.id') ), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer(), ForeignKey('revision.id') ), - ) - - package_revision = Table('package_revision', meta, - Column('id', Integer(), primary_key=True, nullable=False), - Column('name', Unicode(100), nullable=False), - Column('title', UnicodeText()), - Column('version', Unicode(100)), - Column('url', UnicodeText()), - Column('download_url', UnicodeText()), - Column('notes', UnicodeText()), - Column('license_id', Integer(), ForeignKey('license.id') ), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer() , ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', Integer(), ForeignKey('package.id') ), - ) - - tag = Table('tag', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('name', Unicode(100), nullable=False, unique=True), - ) - - package_tag = Table('package_tag', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('package_id', Integer(), ForeignKey('package.id') ), - Column('tag_id', Integer(), ForeignKey('tag.id') ), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer(), ForeignKey('revision.id') ), - ) - - package_tag_revision = Table('package_tag_revision', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('package_id', Integer(), ForeignKey('package.id') ), - Column('tag_id', Integer(), ForeignKey('tag.id') ), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer() , ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', Integer(), ForeignKey('package_tag.id') ), - ) - - package_extra = Table('package_extra', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('package_id', Integer(), ForeignKey('package.id') ), - Column('key', UnicodeText()), - Column('value', UnicodeText()), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer(), ForeignKey('revision.id') ), - ) - - package_extra_revision = Table('package_extra_revision', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('package_id', Integer(), ForeignKey('package.id') ), - Column('key', UnicodeText()), - Column('value', UnicodeText()), - Column('state_id', Integer(), ForeignKey('state.id') ), - Column('revision_id', Integer(), ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', Integer(), ForeignKey('package_extra.id') ), - ) - - meta.bind = migrate_engine - meta.create_all() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py b/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py new file mode 100644 index 00000000000..b31b01e7daa --- /dev/null +++ b/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py @@ -0,0 +1,34 @@ +"""Add author and maintainer + +Revision ID: 86fdd8c54775 +Revises: 103676e0a497 +Create Date: 2018-09-04 17:11:59.181744 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '86fdd8c54775' +down_revision = '103676e0a497' +branch_labels = None +depends_on = None + +_columns = ( + 'author', + 'author_email', + 'maintainer', + 'maintainer_email', +) + + +def upgrade(): + for column in _columns: + op.add_column('package', sa.Column(column, sa.UnicodeText)) + op.add_column('package_revision', sa.Column(column, sa.UnicodeText)) + + +def downgrade(): + for column in reversed(_columns): + op.drop_column('package_revision', column) + op.drop_column('package', column) diff --git a/ckan/migration/versions/002_add_author_and_maintainer.py b/ckan/migration/versions/002_add_author_and_maintainer.py deleted file mode 100644 index 4178e24ff6e..00000000000 --- a/ckan/migration/versions/002_add_author_and_maintainer.py +++ /dev/null @@ -1,34 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - # Upgrade operations go here. Don't create your own engine; use the engine - # named 'migrate_engine' imported from migrate. - package = Table('package', metadata, autoload=True) - package_revision = Table('package_revision', metadata, autoload=True) - - columns = [Column('author', UnicodeText), - Column('author_email', UnicodeText), - Column('maintainer', UnicodeText), - Column('maintainer_email', UnicodeText), - ] - columns2 = [Column('author', UnicodeText), - Column('author_email', UnicodeText), - Column('maintainer', UnicodeText), - Column('maintainer_email', UnicodeText), - ] - - for col in columns: - col.create(package) - for col in columns2: - col.create(package_revision) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/003_add_user_object.py b/ckan/migration/versions/003_add_user_object.py deleted file mode 100644 index f81529dd631..00000000000 --- a/ckan/migration/versions/003_add_user_object.py +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - - -def upgrade(migrate_engine): - metadata = MetaData() - user_table = Table('user', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText), - Column('apikey', UnicodeText, default=make_uuid) - ) - metadata.bind = migrate_engine - apikey_table = Table('apikey', metadata, autoload=True) - - user_table.create() - apikey_table.drop() - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py b/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py new file mode 100644 index 00000000000..d874d014d2c --- /dev/null +++ b/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py @@ -0,0 +1,34 @@ +"""Add user object + +Revision ID: f22b4f5241a5 +Revises: 86fdd8c54775 +Create Date: 2018-09-04 17:19:32.836747 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'f22b4f5241a5' +down_revision = '86fdd8c54775' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'user', sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('name', sa.UnicodeText), sa.Column('apikey', sa.UnicodeText) + ) + + op.drop_table('apikey') + + +def downgrade(): + op.create_table( + 'apikey', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.UnicodeText()), + sa.Column('key', sa.UnicodeText()), + ) + op.drop_table('user') diff --git a/ckan/migration/versions/004_add_group_object.py b/ckan/migration/versions/004_add_group_object.py deleted file mode 100644 index 5d1c3d414d1..00000000000 --- a/ckan/migration/versions/004_add_group_object.py +++ /dev/null @@ -1,38 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - # you need to load this for foreign keys to work in package_group_table - package_table = Table('package', metadata, autoload=True) - - group_table = Table('group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText, unique=True, nullable=False), - Column('title', UnicodeText), - Column('description', UnicodeText), - ) - - package_group_table = Table('package_group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', Integer, ForeignKey('package.id')), - Column('group_id', UnicodeText, ForeignKey('group.id')), - ) - - - group_table.create() - package_group_table.create() - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/004_f92ee205e46d_add_group_object.py b/ckan/migration/versions/004_f92ee205e46d_add_group_object.py new file mode 100644 index 00000000000..fb833544ef9 --- /dev/null +++ b/ckan/migration/versions/004_f92ee205e46d_add_group_object.py @@ -0,0 +1,40 @@ +"""Add group object + +Revision ID: f92ee205e46d +Revises: f22b4f5241a5 +Create Date: 2018-09-04 17:22:50.675045 + +""" +import uuid + +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'f92ee205e46d' +down_revision = 'f22b4f5241a5' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.create_table( + 'group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('name', sa.UnicodeText, unique=True, nullable=False), + sa.Column('title', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + ) + + op.create_table( + 'package_group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + ) + + +def downgrade(): + op.drop_table('package_group') + op.drop_table('group') diff --git a/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py b/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py new file mode 100644 index 00000000000..db146351be5 --- /dev/null +++ b/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py @@ -0,0 +1,61 @@ +"""005 Add authorization tables + +Revision ID: 12c2232c15f5 +Revises: f92ee205e46d +Create Date: 2018-09-04 17:34:59.667587 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '12c2232c15f5' +down_revision = 'f92ee205e46d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'role_action', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('role', sa.UnicodeText), + sa.Column('context', sa.UnicodeText, nullable=False), + sa.Column('action', sa.UnicodeText), + ) + + op.create_table( + 'user_object_role', sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('user_id', sa.UnicodeText, sa.ForeignKey('user.id')), + sa.Column('context', sa.UnicodeText, nullable=False), + sa.Column('role', sa.UnicodeText) + ) + + op.create_table( + 'package_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + ) + + op.create_table( + 'group_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + ) + + +def downgrade(): + op.drop_table('group_role') + op.drop_table('package_role') + op.drop_table('user_object_role') + op.drop_table('role_action') diff --git a/ckan/migration/versions/005_add_authorization_tables.py b/ckan/migration/versions/005_add_authorization_tables.py deleted file mode 100644 index e73b72f88d4..00000000000 --- a/ckan/migration/versions/005_add_authorization_tables.py +++ /dev/null @@ -1,57 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - # you need to load these two for foreign keys to work - package_table = Table('package', metadata, autoload=True) - group_table = Table('group', metadata, autoload=True) - user_table = Table('user', metadata, autoload=True) - - # authorization tables - role_action_table = Table('role_action', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('role', UnicodeText), - Column('context', UnicodeText, nullable=False), - Column('action', UnicodeText), - ) - - user_object_role_table = Table('user_object_role', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('user_id', UnicodeText, ForeignKey('user.id')), - Column('context', UnicodeText, nullable=False), - Column('role', UnicodeText) - ) - - package_role_table = Table('package_role', metadata, - Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True), - Column('package_id', Integer, ForeignKey('package.id')), - ) - - group_role_table = Table('group_role', metadata, - Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True), - Column('group_id', UnicodeText, ForeignKey('group.id')), - ) - - role_action_table.create() - user_object_role_table.create() - package_role_table.create() - group_role_table.create() - - # Not Good to use model here. TODO: refactor this. - # import ckan.model as model - # model.repo.init_db() - # call this explicitly elsewhere - # model.give_all_packages_default_user_roles() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/006_add_ratings.py b/ckan/migration/versions/006_add_ratings.py deleted file mode 100644 index 4af295f6577..00000000000 --- a/ckan/migration/versions/006_add_ratings.py +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - # you need to load these two for foreign keys to work - package_table = Table('package', metadata, autoload=True) - user_table = Table('user', metadata, autoload=True) - - rating_table = Table('rating', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('user_id', UnicodeText, ForeignKey('user.id')), - Column('user_ip_address', UnicodeText), # alternative to user_id if not logged in - Column('package_id', Integer, ForeignKey('package.id')), - Column('rating', Float) - ) - - rating_table.create() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/006_c83955e7acb6_add_ratings.py b/ckan/migration/versions/006_c83955e7acb6_add_ratings.py new file mode 100644 index 00000000000..b71585cd76e --- /dev/null +++ b/ckan/migration/versions/006_c83955e7acb6_add_ratings.py @@ -0,0 +1,31 @@ +"""006 Add ratings + +Revision ID: c83955e7acb6 +Revises: 12c2232c15f5 +Create Date: 2018-09-04 17:39:11.520922 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'c83955e7acb6' +down_revision = '12c2232c15f5' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'rating', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('user_id', sa.UnicodeText, sa.ForeignKey('user.id')), + sa.Column('user_ip_address', + sa.UnicodeText), # alternative to user_id if not logged in + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('rating', sa.Float) + ) + + +def downgrade(): + op.drop_table('rating') diff --git a/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py b/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py new file mode 100644 index 00000000000..dafc500053e --- /dev/null +++ b/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py @@ -0,0 +1,31 @@ +"""007 Add system roles + +Revision ID: 1928d4af1cda +Revises: c83955e7acb6 +Create Date: 2018-09-04 17:42:00.367475 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '1928d4af1cda' +down_revision = 'c83955e7acb6' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'system_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + ) + + +def downgrade(): + op.drop_table('system_role') diff --git a/ckan/migration/versions/007_add_system_roles.py b/ckan/migration/versions/007_add_system_roles.py deleted file mode 100644 index d7fe8094be8..00000000000 --- a/ckan/migration/versions/007_add_system_roles.py +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - user_object_role_table = Table('user_object_role', metadata, autoload=True) - - # authorization tables - system_role_table = Table('system_role', metadata, - Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True), - ) - - system_role_table.create() - - # you can now add system administrators - # e.g. paster create-sysadmin http://bgates.openid.com/ - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py b/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py new file mode 100644 index 00000000000..02e4850ce80 --- /dev/null +++ b/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py @@ -0,0 +1,53 @@ +"""008 Update vdm ids + +Revision ID: e8283ffb257e +Revises: 1928d4af1cda +Create Date: 2018-09-04 17:43:27.042436 + +Originally this revision was altering id columns +to UnicodeText +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e8283ffb257e' +down_revision = '1928d4af1cda' +branch_labels = None +depends_on = None + +foreign_tables = ( + 'package', 'package_tag', 'package_revision', 'package_tag_revision', + 'package_extra', 'package_extra_revision' +) + + +def upgrade(): + for table in foreign_tables: + op.drop_column(table, 'revision_id') + op.alter_column('revision', 'id', type_=sa.UnicodeText, server_default=None) + for table in foreign_tables: + op.add_column( + table, + sa.Column( + 'revision_id', sa.UnicodeText, sa.ForeignKey('revision.id') + ) + ) + + +def downgrade(): + for table in foreign_tables: + op.drop_column(table, 'revision_id') + op.drop_column('revision', 'id') + op.add_column( + 'revision', + sa.Column( + 'id', sa.Integer, primary_key=True, nullable=False, unique=True + ) + ) + + for table in foreign_tables: + op.add_column( + table, + sa.Column('revision_id', sa.Integer, sa.ForeignKey('revision.id')) + ) diff --git a/ckan/migration/versions/008_update_vdm_ids.py b/ckan/migration/versions/008_update_vdm_ids.py deleted file mode 100644 index f6a2c74b0bb..00000000000 --- a/ckan/migration/versions/008_update_vdm_ids.py +++ /dev/null @@ -1,91 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -import sqlalchemy.schema -import uuid - -from migrate import * -import migrate.changeset -from migrate.changeset.constraint import ForeignKeyConstraint, PrimaryKeyConstraint - -metadata = None - -def make_uuid(): - return unicode(uuid.uuid4()) - -# 1 drop all foreign key constraints -# 2 alter type of revision id and foreign keys -# 3 create foreign key constraints (using cascade!) -# 4 create uuids for revisions (auto cascades elsewhere!) -def upgrade(migrate_engine): - global metadata - metadata = MetaData() - metadata.bind = migrate_engine - dropped_fk_constraints = drop_constraints_and_alter_types() - upgrade2(migrate_engine, dropped_fk_constraints) - -def drop_constraints_and_alter_types(): - # 1 drop all foreign key constraints - dropped_fk_constraints = [] - revision_table = Table('revision', metadata, autoload=True) - foreign_tables = ['package', 'package_tag', 'package_revision', 'package_tag_revision', 'package_extra', 'package_extra_revision', ] - for table_name in foreign_tables: - table = Table(table_name, metadata, autoload=True) - for constraint in table.constraints.copy(): - if isinstance(constraint, sqlalchemy.schema.ForeignKeyConstraint): - foreign_key_cols = [key.column for key in constraint.elements] - fk_col = foreign_key_cols[0] - if fk_col.table == revision_table: - orig_fk = ForeignKeyConstraint(constraint.columns, foreign_key_cols, name=constraint.name, table=table) - orig_fk.drop() - dropped_fk_constraints.append((constraint.columns, foreign_key_cols, constraint.name, table.name)) - - # 2 alter type of revision id and foreign keys - # Sanity check - if len(constraint.columns.keys()) != 1: - raise ValueError() - id_col = constraint.table.columns[constraint.columns.keys()[0]] - id_col.alter(type=UnicodeText) - - revision_table = Table('revision', metadata, autoload=True) - id_col = revision_table.c['id'] - id_col.alter(type=UnicodeText, - ) - - return dropped_fk_constraints - -def upgrade2(migrate_engine, dropped_fk_constraints): - # have changed type of cols so recreate metadata - metadata = MetaData(migrate_engine) - - # 3 create foreign key constraints - for fk_constraint in dropped_fk_constraints: - # cascade doesn't work - # see http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=48 - # new_fk = ForeignKeyConstraint(*fk_constraint, onupdate='CASCADE') - # new_fk = ForeignKeyConstraint(*fk_constraint) - # new_fk.create() - - # So we create via hand ... - constraint_columns, foreign_key_cols, constraint_name, table_name = fk_constraint - # Sanity check - if len(constraint_columns.keys()) != 1: - raise ValueError() - - oursql = '''ALTER TABLE %(table)s - ADD CONSTRAINT %(fkeyname)s - FOREIGN KEY (%(col_name)s) - REFERENCES revision (id) - ''' % {'table':table_name, 'fkeyname':constraint_name, - 'col_name':constraint_columns.keys()[0] } - migrate_engine.execute(oursql) - - # 4 create uuids for revisions and in related tables - revision_table = Table('revision', metadata, autoload=True) - from sqlalchemy.sql import select - for row in migrate_engine.execute(select([revision_table])): - update = revision_table.update().where(revision_table.c.id==row.id).values(id=make_uuid()) - migrate_engine.execute(update) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/009_add_creation_timestamps.py b/ckan/migration/versions/009_add_creation_timestamps.py deleted file mode 100644 index beb83d23bad..00000000000 --- a/ckan/migration/versions/009_add_creation_timestamps.py +++ /dev/null @@ -1,27 +0,0 @@ -# encoding: utf-8 - -from datetime import datetime - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -domain_obj_names = ['rating', 'group', 'user'] - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - # Use sql instead of migrate.changeset because user and group are sql - # reserved words and migrate doesn't quote them. - for domain_obj_name in domain_obj_names: - sql = 'ALTER TABLE "%s" ADD created TIMESTAMP WITHOUT TIME ZONE' % domain_obj_name - migrate_engine.execute(sql) - - now = datetime.now() - for domain_obj_name in domain_obj_names[::-1]: - table = Table(domain_obj_name, metadata, autoload=True) - migrate_engine.execute(table.update(values={table.c.created:now})) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py b/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py new file mode 100644 index 00000000000..a1574896d08 --- /dev/null +++ b/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py @@ -0,0 +1,27 @@ +"""009 Add creation timestamps + +Revision ID: b739a48de5c4 +Revises: e8283ffb257e +Create Date: 2018-09-04 18:41:15.442929 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b739a48de5c4' +down_revision = 'e8283ffb257e' +branch_labels = None +depends_on = None + +domain_obj_names = ['rating', 'group', 'user'] + + +def upgrade(): + for table in domain_obj_names: + op.add_column(table, sa.Column('created', sa.TIMESTAMP)) + + +def downgrade(): + for table in domain_obj_names: + op.drop_column(table, 'created') diff --git a/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py b/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py new file mode 100644 index 00000000000..d791cda98c4 --- /dev/null +++ b/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py @@ -0,0 +1,23 @@ +"""010 Add user about + +Revision ID: a6f13bf14d0c +Revises: b739a48de5c4 +Create Date: 2018-09-04 18:44:53.313230 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'a6f13bf14d0c' +down_revision = 'b739a48de5c4' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('user', sa.Column('about', sa.Text)) + + +def downgrade(): + op.drop_column('user', 'about') diff --git a/ckan/migration/versions/010_add_user_about.py b/ckan/migration/versions/010_add_user_about.py deleted file mode 100644 index c4f3c10dfac..00000000000 --- a/ckan/migration/versions/010_add_user_about.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - # Using sql because migrate doesn't quote reserved word 'user' - user_sql = 'ALTER TABLE "user" ADD about TEXT' - migrate_engine.execute(user_sql) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py b/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py new file mode 100644 index 00000000000..cb337f0b158 --- /dev/null +++ b/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py @@ -0,0 +1,31 @@ +"""011 Add package search vector + +Revision ID: 866f6370b4ac +Revises: 746205dde53d +Create Date: 2018-09-04 18:48:51.971937 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '866f6370b4ac' +down_revision = 'a6f13bf14d0c' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'package_search', + sa.Column( + 'package_id', + sa.Integer, + sa.ForeignKey('package.id'), + primary_key=True + ), sa.Column('search_vector', sa.dialects.postgresql.TSVECTOR) + ) + + +def downgrade(): + op.drop_table('package_search') diff --git a/ckan/migration/versions/011_add_package_search_vector.py b/ckan/migration/versions/011_add_package_search_vector.py deleted file mode 100644 index ff9eea3e5c3..00000000000 --- a/ckan/migration/versions/011_add_package_search_vector.py +++ /dev/null @@ -1,25 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - package_table = Table('package', metadata, autoload=True) - package_search_table = Table('package_search', metadata, - Column('package_id', Integer, ForeignKey('package.id'), primary_key=True), - ) - - package_search_table.create() - sql = 'ALTER TABLE package_search ADD COLUMN search_vector tsvector' - migrate_engine.execute(sql) - - # This is not so important now and annoying to read when testing - #print 'IMPORTANT! Now run:\n paster create-search-index' - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/012_add_resources.py b/ckan/migration/versions/012_add_resources.py deleted file mode 100644 index 2ebf11b21ed..00000000000 --- a/ckan/migration/versions/012_add_resources.py +++ /dev/null @@ -1,80 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset -import vdm.sqlalchemy - - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - package_table = Table('package', metadata, autoload=True) - package_revision_table = Table('package_revision', metadata, autoload=True) - package_resource_table = Table('package_resource', metadata, - Column('id', Integer, primary_key=True), - Column('package_id', Integer, ForeignKey('package.id')), - Column('url', UnicodeText, nullable=False), - Column('format', UnicodeText), - Column('description', UnicodeText), - Column('position', Integer), - Column('state_id', Integer), - Column('revision_id', UnicodeText, ForeignKey('revision.id')) - ) - - package_resource_revision_table = Table('package_resource_revision', metadata, - Column('id', Integer, primary_key=True), - Column('package_id', Integer, ForeignKey('package.id')), - Column('url', UnicodeText, nullable=False), - Column('format', UnicodeText), - Column('description', UnicodeText), - Column('position', Integer), - Column('state_id', Integer), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', Integer, ForeignKey('package_resource.id')) - ) - - - package_resource_table.create() - package_resource_revision_table.create() - - # Move download_urls across to resources - # NB: strictly we should check each package_revision to check whether - # download_url changed (and if only change) and then create - # package_resource_revision for each such revision (and delete every - # package_revision where only change is download_url) - # However, we adopt a cruder approach in which we just create - engine = migrate_engine - select_sql = select([package_table]) - for pkg in engine.execute(select_sql): - download_url = pkg['download_url'] - if download_url: - # what about revision_id? - res_values = {'package_id':pkg.id, - 'url':download_url, - 'format':u'', - 'description':u'', - 'position':0, - 'state_id':1, - 'revision_id': pkg.revision_id, - } - insert_sql = package_resource_table.insert(values=res_values) - engine.execute(insert_sql) - # get id of just inserted resource - getid_sql = select([package_resource_table]).where( - package_resource_table.c.package_id==pkg.id) - resource_id = getid_sql.execute().fetchone().id - # now we need to update revision table ... - res_rev_values = dict(res_values) - res_rev_values['continuity_id'] = resource_id - res_rev_values['id'] = resource_id - insert_sql = package_resource_revision_table.insert(values=res_rev_values) - engine.execute(insert_sql) - - package_table.c.download_url.drop() - package_revision_table.c.download_url.drop() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/012_e5ca33a5d445_add_resources.py b/ckan/migration/versions/012_e5ca33a5d445_add_resources.py new file mode 100644 index 00000000000..bcb2ef9c80a --- /dev/null +++ b/ckan/migration/versions/012_e5ca33a5d445_add_resources.py @@ -0,0 +1,63 @@ +"""012 Add resources + +Revision ID: e5ca33a5d445 +Revises: 866f6370b4ac +Create Date: 2018-09-04 18:48:52.303211 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e5ca33a5d445' +down_revision = '866f6370b4ac' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.create_table( + 'package_resource', sa.Column('id', sa.Integer, primary_key=True), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('url', sa.UnicodeText, nullable=False), + sa.Column('format', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column('position', sa.Integer), sa.Column('state_id', sa.Integer), + sa.Column('revision_id', sa.UnicodeText, sa.ForeignKey('revision.id')) + ) + + op.create_table( + 'package_resource_revision', + sa.Column('id', sa.Integer, primary_key=True), + sa.Column('package_id', sa.Integer, sa.ForeignKey('package.id')), + sa.Column('url', sa.UnicodeText, nullable=False), + sa.Column('format', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column('position', sa.Integer), sa.Column('state_id', sa.Integer), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.Integer, sa.ForeignKey('package_resource.id') + ) + ) + + op.drop_column('package', 'download_url') + op.drop_column('package_revision', 'download_url') + + +def downgrade(): + op.drop_table('package_resource_revision') + op.drop_table('package_resource') + op.add_column( + 'package', + sa.Column('download_url', sa.UnicodeText()), + ) + op.add_column( + 'package_revision', + sa.Column('download_url', sa.UnicodeText()), + ) diff --git a/ckan/migration/versions/013_8a3a5af39797_add_hash.py b/ckan/migration/versions/013_8a3a5af39797_add_hash.py new file mode 100644 index 00000000000..0c288e12314 --- /dev/null +++ b/ckan/migration/versions/013_8a3a5af39797_add_hash.py @@ -0,0 +1,23 @@ +"""013 Add hash + +Revision ID: 8a3a5af39797 +Revises: e5ca33a5d445 +Create Date: 2018-09-04 18:48:52.640250 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '8a3a5af39797' +down_revision = 'e5ca33a5d445' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('package_resource', sa.Column('hash', sa.UnicodeText)) + + +def downgrade(): + op.drop_column('package_resource', 'hash') diff --git a/ckan/migration/versions/013_add_hash.py b/ckan/migration/versions/013_add_hash.py deleted file mode 100644 index 68bb54184fe..00000000000 --- a/ckan/migration/versions/013_add_hash.py +++ /dev/null @@ -1,17 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - package_resource = Table('package_resource', metadata, autoload=True) - column = Column('hash', UnicodeText) - column.create(package_resource) - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/014_93519b684820_hash_2.py b/ckan/migration/versions/014_93519b684820_hash_2.py new file mode 100644 index 00000000000..d9f16989de1 --- /dev/null +++ b/ckan/migration/versions/014_93519b684820_hash_2.py @@ -0,0 +1,25 @@ +"""014 Hash 2 + +Revision ID: 93519b684820 +Revises: 8a3a5af39797 +Create Date: 2018-09-04 18:48:52.968191 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '93519b684820' +down_revision = '8a3a5af39797' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'package_resource_revision', sa.Column('hash', sa.UnicodeText) + ) + + +def downgrade(): + op.drop_column('package_resource_revision', 'hash') diff --git a/ckan/migration/versions/014_hash_2.py b/ckan/migration/versions/014_hash_2.py deleted file mode 100644 index 8a1e320248c..00000000000 --- a/ckan/migration/versions/014_hash_2.py +++ /dev/null @@ -1,19 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - # When adding a column to a revisioned object, need to add it to it's - # counterpart revision object too. Here is the counter-part for that in - # 013_add_hash.py - package_resource_revision = Table('package_resource_revision', metadata, autoload=True) - column = Column('hash', UnicodeText) - column.create(package_resource_revision) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py b/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py new file mode 100644 index 00000000000..9d1988fefd0 --- /dev/null +++ b/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py @@ -0,0 +1,41 @@ +"""015 Remove state_object + +Revision ID: 6d8ffebcaf54 +Revises: 93519b684820 +Create Date: 2018-09-04 18:48:53.302758 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6d8ffebcaf54' +down_revision = '93519b684820' +branch_labels = None +depends_on = None + +stateful_tables = [ + 'license', 'package', 'package_revision', 'package_tag', + 'package_tag_revision', 'package_extra', 'package_extra_revision', + 'package_resource', 'package_resource_revision', 'revision' +] + + +def upgrade(): + for table_name in stateful_tables: + op.add_column(table_name, sa.Column('state', sa.UnicodeText)) + op.drop_column(table_name, 'state_id') + + op.drop_table('state') + + +def downgrade(): + op.create_table( + 'state', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100)), + ) + + for table_name in stateful_tables: + op.drop_column(table_name, 'state') + op.add_column(table_name, sa.Column('state_id', sa.Integer)) diff --git a/ckan/migration/versions/015_remove_state_object.py b/ckan/migration/versions/015_remove_state_object.py deleted file mode 100644 index 0c0eda373b7..00000000000 --- a/ckan/migration/versions/015_remove_state_object.py +++ /dev/null @@ -1,35 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -import sqlalchemy.sql as sql - -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - stateful_tables = [ - 'license', - 'package', 'package_revision', - 'package_tag', 'package_tag_revision', - 'package_extra', 'package_extra_revision', - 'package_resource', 'package_resource_revision', - 'revision' - ] - for table_name in stateful_tables: - # print '***** Processing table: %s' % table_name - table = Table(table_name, metadata, autoload=True) - column = Column('state', UnicodeText) - column.create(table) - for (name,id) in [ ('active',1), ('deleted',2), ('pending',3) ]: - sqlcmd = '''UPDATE %s SET state = '%s' WHERE state_id = %s''' % (table.name, name, id) - migrate_engine.execute(sqlcmd) - stateid = table.c['state_id'] - stateid.drop() - table = Table('state', metadata, autoload=True) - table.drop() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py b/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py new file mode 100644 index 00000000000..75ef15f8591 --- /dev/null +++ b/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py @@ -0,0 +1,127 @@ +"""016 Uuids everywhere + +Revision ID: 37ada738328e +Revises: 6d8ffebcaf54 +Create Date: 2018-09-04 18:48:53.632517 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '37ada738328e' +down_revision = '6d8ffebcaf54' +branch_labels = None +depends_on = None + +foreign_keys = [ + ('package_tag', 'package_id'), + ('package_tag', 'tag_id'), + ('package_extra', 'package_id'), + ('package_resource', 'package_id'), + ('package_tag_revision', 'package_id'), + ('package_tag_revision', 'tag_id'), + ('package_extra_revision', 'package_id'), + ('package_resource_revision', 'package_id'), + ('rating', 'package_id'), + ('package_search', 'package_id'), + ('package_role', 'package_id'), + ('package_group', 'package_id'), +] +ids = [ + 'package', 'package_tag', 'package_extra', 'package_resource', + 'package_revision', 'package_tag_revision', 'package_extra_revision', + 'package_resource_revision', 'tag' +] + +continuity = [ + 'package_revision', 'package_tag_revision', 'package_extra_revision', + 'package_resource_revision' +] + +sequences = [ + 'package_id_seq', 'package_extra_id_seq', 'package_resource_id_seq', + 'package_tag_id_seq', 'tag_id_seq' +] + +combined_primary_keys = [ + 'package_extra_revision', 'package_revision', 'package_tag_revision' +] + + +def upgrade(): + for table, column in foreign_keys: + op.drop_column(table, column) + for table in continuity: + op.drop_column(table, 'continuity_id') + for table in ids: + op.alter_column(table, 'id', type_=sa.UnicodeText, server_default=None) + + for table, column in foreign_keys: + op.add_column( + table, + sa.Column( + column, sa.UnicodeText, + sa.ForeignKey(column.replace('_', '.')) + ) + ) + for table in continuity: + op.add_column( + table, + sa.Column( + 'continuity_id', sa.UnicodeText, + sa.ForeignKey(table.replace('_revision', '.id')) + ) + ) + for table in combined_primary_keys: + op.create_primary_key(table + '_pkey', table, ['id', 'revision_id']) + + op.execute( + sa.schema.DropSequence(sa.schema.Sequence('package_extra_id_seq')) + ) + op.execute(sa.schema.DropSequence(sa.schema.Sequence('package_id_seq'))) + op.execute( + sa.schema.DropSequence(sa.schema.Sequence('package_tag_id_seq')) + ) + op.execute( + sa.schema.DropSequence(sa.schema.Sequence('package_resource_id_seq')) + ) + op.execute(sa.schema.DropSequence(sa.schema.Sequence('tag_id_seq'))) + op.execute(sa.schema.DropSequence(sa.schema.Sequence('revision_id_seq'))) + + +def downgrade(): + for table, column in foreign_keys: + op.drop_column(table, column) + for table in continuity: + op.drop_column(table, 'continuity_id') + + for table in ids: + op.drop_column(table, 'id') + op.add_column( + table, + sa.Column( + 'id', + sa.Integer, + primary_key=True, + autoincrement=True, + nullable=False, + unique=True + ) + ) + + for table, column in foreign_keys: + op.add_column( + table, + sa.Column( + column, sa.Integer, sa.ForeignKey(column.replace('_', '.')) + ) + ) + for table in continuity: + op.add_column( + table, + sa.Column( + 'continuity_id', sa.Integer, + sa.ForeignKey(table.replace('_revision', '.id')) + ) + ) diff --git a/ckan/migration/versions/016_uuids_everywhere.py b/ckan/migration/versions/016_uuids_everywhere.py deleted file mode 100644 index d661268fd6c..00000000000 --- a/ckan/migration/versions/016_uuids_everywhere.py +++ /dev/null @@ -1,196 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -import sqlalchemy.schema -import uuid -from sqlalchemy.sql import select - -from migrate import * -import migrate.changeset -from migrate.changeset.constraint import ForeignKeyConstraint, PrimaryKeyConstraint - -metadata = MetaData() - -def make_uuid(): - return unicode(uuid.uuid4()) - -## Tables and columns changed in the model -## Versioned: -## ('package', 'id'), -## ('package_tag', ('id', 'package_id', 'tag_id')), -## ('package_extra', ('id', 'package_id')), -## ('package_resource', ('id', 'package_id')), -## Versions: -## ('package_revision', 'id'), -## ('package_tag_revision', ('id', 'package_id', 'tag_id')), -## ('package_extra_revision', ('id', 'package_id')), -## ('package_resource_revision', ('id', 'package_id')), -## Non-versioned: -## ('tag', 'id'), -## ('rating', 'package_id'), -## ('package_search', 'package_id'), -## ('package_role', 'package_id'), -## ('package_group', 'package_id'), - -def upgrade(migrate_engine): - global metadata - metadata = MetaData() - metadata.bind = migrate_engine - primary_table_name = 'package' - foreign_tables = ['package_revision', - 'package_tag', 'package_tag_revision', - 'package_extra', 'package_extra_revision', - 'package_resource', 'package_resource_revision', - 'rating', 'package_search', - 'package_role', 'package_group'] - revision_table_name = 'package_revision' - convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name) - - primary_table_name = 'package_resource' - foreign_tables = ['package_resource_revision'] - revision_table_name = 'package_resource_revision' - convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name) - - primary_table_name = 'package_tag' - foreign_tables = ['package_tag_revision'] - revision_table_name = 'package_tag_revision' - convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name) - - primary_table_name = 'package_extra' - foreign_tables = ['package_extra_revision'] - revision_table_name = 'package_extra_revision' - convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name) - - primary_table_name = 'tag' - foreign_tables = ['package_tag', 'package_tag_revision'] - revision_table_name = None - convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name) - - drop_sequencies(migrate_engine) - -def convert_to_uuids(migrate_engine, primary_table_name, foreign_tables, revision_table_name=None): - '''Convert an id column in Primary Table to string type UUIDs. - How it works: - 1 drop all foreign key constraints - 2 alter type of revision id and foreign keys - 3 create foreign key constraints (using cascade!) - 4 create uuids for revisions (auto cascades elsewhere!) - - @param primary_table_name - table containing the primary key id column - @param foreign_tables - names of tables which have this same key as a - foreign key constraint - @param revision_table_name - if primary_table is versioned, supply the name - of its related revision table, so that it can be updated at the same - time. - ''' - #print('** Processing %s' % primary_table_name) - #print('*** Dropping fk constraints') - dropped_fk_constraints = drop_constraints_and_alter_types(primary_table_name, foreign_tables, revision_table_name) - #print('*** Adding fk constraints (with cascade)') - add_fk_constraints(migrate_engine, dropped_fk_constraints, primary_table_name) - #print('*** Creating UUIDs') - create_uuids(migrate_engine, primary_table_name, revision_table_name) - -def drop_constraints_and_alter_types(primary_table_name, foreign_tables, revision_table_name): - # 1 drop all foreign key constraints - dropped_fk_constraints = [] - primary_table = Table(primary_table_name, metadata, autoload=True) - for table_name in foreign_tables: - table = Table(table_name, metadata, autoload=True) - for constraint in table.constraints.copy(): - if isinstance(constraint, sqlalchemy.schema.ForeignKeyConstraint): - foreign_key_cols = [key.column for key in constraint.elements] - fk_col = foreign_key_cols[0] - if fk_col.table == primary_table: - orig_fk = ForeignKeyConstraint(constraint.columns, foreign_key_cols, name=constraint.name, table=table) - orig_fk.drop() - dropped_fk_constraints.append((constraint.columns, foreign_key_cols, constraint.name, table.name)) - #print 'CON', dropped_fk_constraints[-1] - - # 2 alter type of primary table id and foreign keys - if len(constraint.columns.keys()) != 1: - raise ValueError() - id_col = constraint.table.columns[constraint.columns.keys()[0]] - id_col.alter(type=UnicodeText) - - primary_table = Table(primary_table_name, metadata, autoload=True) - id_col = primary_table.c['id'] - id_col.alter(type=UnicodeText) - - if revision_table_name: - # Revision table id column type changed as well - revision_table = Table(revision_table_name, metadata, autoload=True) - id_col = revision_table.c['id'] - id_col.alter(type=UnicodeText) - - return dropped_fk_constraints - -def add_fk_constraints(migrate_engine, dropped_fk_constraints, primary_table_name): - # 3 create foreign key constraints - for fk_constraint in dropped_fk_constraints: - # cascade doesn't work - # see http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=48 - # new_fk = ForeignKeyConstraint(*fk_constraint, onupdate='CASCADE') - # new_fk = ForeignKeyConstraint(*fk_constraint) - # new_fk.create() - - # So we create via hand ... - constraint_columns, foreign_key_cols, constraint_name, table_name = fk_constraint - # Sanity check - if len(constraint_columns.keys()) != 1: - raise ValueError() - - oursql = '''ALTER TABLE %(table)s - ADD CONSTRAINT %(fkeyname)s - FOREIGN KEY (%(col_name)s) - REFERENCES %(primary_table_name)s (id) - ''' % {'table':table_name, 'fkeyname':constraint_name, - 'col_name':constraint_columns.keys()[0], - 'primary_table_name':primary_table_name} - migrate_engine.execute(oursql) - -def create_uuids(migrate_engine, primary_table_name, revision_table_name): - # have changed type of cols so recreate metadata - metadata = MetaData(migrate_engine) - - # 4 create uuids for primary entities and in related tables - primary_table = Table(primary_table_name, metadata, autoload=True) - if revision_table_name: - revision_table = Table(revision_table_name, metadata, autoload=True) - # fetchall wouldn't be optimal with really large sets of data but here <20k - ids = [ res[0] for res in - migrate_engine.execute(select([primary_table.c.id])).fetchall() ] - for count,id in enumerate(ids): - # if count % 100 == 0: print(count, id) - myuuid = make_uuid() - update = primary_table.update().where(primary_table.c.id==id).values(id=myuuid) - migrate_engine.execute(update) - if revision_table_name: - # ensure each id in revision table match its continuity id. - q = revision_table.update().values(id=revision_table.c.continuity_id) - migrate_engine.execute(q) - - -def drop_sequencies(migrate_engine): - - sequences = ['package_extra', 'package_extra_revision', 'package', - 'package_resource', 'package_resource_revision', - 'package_revision', ' package_tag', 'package_tag_revision', - 'revision', 'tag'] - - for sequence in sequences: - migrate_engine.execute('ALTER TABLE %s ALTER COLUMN id DROP DEFAULT;' % sequence) - - for sequence in sequences: - try: - migrate_engine.execute('drop sequence %s_id_seq;' % sequence) - except sqlalchemy.exc.ProgrammingError as e: - if 'sequence "{}_id_seq" does not exist'.format(sequence) in str(e): - pass - else: - raise - - - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py b/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py new file mode 100644 index 00000000000..767f9feb7ac --- /dev/null +++ b/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py @@ -0,0 +1,58 @@ +"""017 Add pkg_relationships + +Revision ID: 1250b2ff3e36 +Revises: 37ada738328e +Create Date: 2018-09-04 18:48:53.963557 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '1250b2ff3e36' +down_revision = '37ada738328e' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.create_table( + 'package_relationship', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column( + 'subject_package_id', sa.UnicodeText, sa.ForeignKey('package.id') + ), + sa.Column( + 'object_package_id', sa.UnicodeText, sa.ForeignKey('package.id') + ), sa.Column('type', sa.UnicodeText), + sa.Column('comment', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, sa.ForeignKey('revision.id')) + ) + + op.create_table( + 'package_relationship_revision', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column( + 'subject_package_id', sa.UnicodeText, sa.ForeignKey('package.id') + ), + sa.Column( + 'object_package_id', sa.UnicodeText, sa.ForeignKey('package.id') + ), sa.Column('type', sa.UnicodeText), + sa.Column('comment', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.UnicodeText, + sa.ForeignKey('package_relationship.id') + ) + ) + + +def downgrade(): + op.drop_table('package_relationship_revision') + op.drop_table('package_relationship') diff --git a/ckan/migration/versions/017_add_pkg_relationships.py b/ckan/migration/versions/017_add_pkg_relationships.py deleted file mode 100644 index f833f40dfae..00000000000 --- a/ckan/migration/versions/017_add_pkg_relationships.py +++ /dev/null @@ -1,41 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - package_table = Table('package', metadata, autoload=True) - - package_relationship_table = Table('package_relationship', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('subject_package_id', UnicodeText, ForeignKey('package.id')), - Column('object_package_id', UnicodeText, ForeignKey('package.id')), - Column('type', UnicodeText), - Column('comment', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id')) - ) - - package_relationship_revision_table = Table('package_relationship_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('subject_package_id', UnicodeText, ForeignKey('package.id')), - Column('object_package_id', UnicodeText, ForeignKey('package.id')), - Column('type', UnicodeText), - Column('comment', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('package_relationship.id')) - ) - - package_relationship_table.create() - package_relationship_revision_table.create() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py b/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py new file mode 100644 index 00000000000..b8c72272284 --- /dev/null +++ b/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py @@ -0,0 +1,40 @@ +"""018 Adjust licenses + +Revision ID: 05a0778051ca +Revises: 1250b2ff3e36 +Create Date: 2018-09-04 18:48:54.288030 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '05a0778051ca' +down_revision = '1250b2ff3e36' +branch_labels = None +depends_on = None + +tables = ['package', 'package_revision'] + + +def upgrade(): + for table in tables: + op.drop_column(table, 'license_id') + op.add_column(table, sa.Column('license_id', sa.UnicodeText)) + op.drop_table('license') + + +def downgrade(): + + op.create_table( + 'license', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('name', sa.Unicode(100)), + sa.Column('state', sa.UnicodeText) + ) + for table in tables: + op.drop_column(table, 'license_id') + op.add_column( + table, + sa.Column('license_id', sa.Integer, sa.ForeignKey('license.id')) + ) diff --git a/ckan/migration/versions/018_adjust_licenses.py b/ckan/migration/versions/018_adjust_licenses.py deleted file mode 100644 index 4b7c1d6e245..00000000000 --- a/ckan/migration/versions/018_adjust_licenses.py +++ /dev/null @@ -1,180 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import uuid - - -map = { - u'OSI Approved::Mozilla Public License 1.1 (MPL)': 'mozilla1.1', - u'OKD Compliant::Creative Commons Attribution-ShareAlike': 'cc-by-sa', - u'OSI Approved::Nokia Open Source License': 'nokia', - u'OSI Approved::Computer Associates Trusted Open Source License 1.1': 'ca-tosl1.1', - u'OKD Compliant::Higher Education Statistics Agency Copyright with data.gov.uk rights': 'hesa-withrights', - u'OSI Approved::Lucent Public License Version 1.02': 'lucent1.02', - u'OSI Approved::Open Software License': 'osl-3.0', - u'OSI Approved::Motosoto License': 'motosoto', - u'OSI Approved::MIT license': 'mit-license', - u'OSI Approved::Mozilla Public License 1.0 (MPL)': 'mozilla', - u'OSI Approved::GNU General Public License v3 (GPLv3)': 'gpl-3.0', - u'OKD Compliant::UK Click Use PSI': 'ukclickusepsi', - u'OSI Approved::Eiffel Forum License': 'eiffel', - u'OSI Approved::Jabber Open Source License': 'jabber-osl', - u'OSI Approved::Open Group Test Suite License': 'opengroup', - u'OSI Approved::Entessa Public License': 'entessa', - u'OKD Compliant::Other': 'other-open', - u'OSI Approved::EU DataGrid Software License': 'eudatagrid', - u'OSI Approved::Zope Public License': 'zpl', - u'OSI Approved::Naumen Public License': 'naumen', - u'OSI Approved::wxWindows Library License': 'wxwindows', - u'OKD Compliant::GNU Free Documentation License (GFDL)': 'gfdl', - u'Non-OKD Compliant::Non-Commercial Other': 'other-nc', - u'OKD Compliant::Open Data Commons Public Domain Dedication and License (PDDL)': 'odc-pddl', - u'OSI Approved::NASA Open Source Agreement 1.3': 'nasa1.3', - u'OSI Approved::X.Net License': 'xnet', - u'OSI Approved::W3C License': 'W3C', - u'OSI Approved::Academic Free License': 'afl-3.0', - u'Non-OKD Compliant::Crown Copyright': 'ukcrown', - u'OSI Approved::RealNetworks Public Source License V1.0': 'real', - u'OSI Approved::Common Development and Distribution License': 'cddl1', - u'OSI Approved::Intel Open Source License': 'intel-osl', - u'OSI Approved::GNU General Public License (GPL)': 'gpl-2.0', - u'Non-OKD Compliant::Creative Commons Non-Commercial (Any)': 'cc-nc', - u'Non-OKD Compliant::Other': 'other-closed', - u'Other::License Not Specified': 'notspecified', - u'OSI Approved::Sybase Open Watcom Public License 1.0': 'sybase', - u'OSI Approved::Educational Community License': 'ecl2', - u'OSI Approved::Sun Industry Standards Source License (SISSL)': 'sun-issl', - u'OKD Compliant::Other (Public Domain)': 'other-pd', - u'OKD Compliant::Public Domain': 'other-pd', - u'OKD Compliant::Creative Commons Attribution': 'cc-by', - u'OSI Approved::OCLC Research Public License 2.0': 'oclc2', - u'OSI Approved::Artistic license': 'artistic-license-2.0', - u'OKD Compliant::Other (Attribution)': 'other-at', - u'OSI Approved::Sleepycat License': 'sleepycat', - u'OSI Approved::PHP License': 'php', - u'OKD Compliant::Creative Commons CCZero': 'cc-zero', - u'OSI Approved::University of Illinois/NCSA Open Source License': 'UoI-NCSA', - u'OSI Approved::Adaptive Public License': 'apl1.0', - u'OSI Approved::Ricoh Source Code Public License': 'ricohpl', - u'OSI Approved::Eiffel Forum License V2.0': 'ver2_eiffel', - u'OSI Approved::Python license (CNRI Python License)': 'pythonpl', - u'OSI Approved::Frameworx License': 'frameworx', - u'OSI Approved::IBM Public License': 'ibmpl', - u'OSI Approved::Fair License': 'fair', - u'OSI Approved::Lucent Public License (Plan9)': 'lucent-plan9', - u'OSI Approved::Nethack General Public License': 'nethack', - u'OSI Approved::Common Public License 1.0': 'cpal_1.0', - u'OSI Approved::Attribution Assurance Licenses': 'attribution', - u'OSI Approved::Reciprocal Public License': 'rpl1.5', - u'OSI Approved::Eclipse Public License': 'eclipse-1.0', - u'OSI Approved::CUA Office Public License Version 1.0': 'cuaoffice', - u'OSI Approved::Vovida Software License v. 1.0': 'vovidapl', - u'OSI Approved::Apple Public Source License': 'apsl-2.0', - u'OKD Compliant::UK Crown Copyright with data.gov.uk rights': 'ukcrown-withrights', - u'OKD Compliant::Local Authority Copyright with data.gov.uk rights': 'localauth-withrights', - u'OKD Compliant::Open Data Commons Open Database License (ODbL)': 'odc-odbl', - u'OSI Approved::New BSD license': 'bsd-license', - u'OSI Approved::Qt Public License (QPL)': 'qtpl', - u'OSI Approved::GNU Library or "Lesser" General Public License (LGPL)': 'lgpl-2.1', - u'OSI Approved::MITRE Collaborative Virtual Workspace License (CVW License)': 'mitre', - u'OSI Approved::Apache License, 2.0': 'apache2.0', - u'OSI Approved::Apache Software License': 'apache', - u'OSI Approved::Python Software Foundation License': 'PythonSoftFoundation', - u'OSI Approved::Sun Public License': 'sunpublic', - u'OSI Approved::zlib/libpng license': 'zlib-license' -} - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - #print("Changing package license_ids to strings.") - - # Get licenses, package license ids, and package revision license ids. - old_license_titles = _get_old_license_titles(migrate_engine) - old_package_license_ids = _get_old_package_license_ids(migrate_engine) - old_package_revision_license_ids = _get_old_package_revision_license_ids(migrate_engine) - _check_map_has_old_license_titles(old_license_titles, map) - - # Upgrade database scheme. - drop_fk_constraint_on_package_table = "ALTER TABLE package DROP CONSTRAINT package_license_id_fkey;" - drop_fk_constraint_on_package_revision_table = "ALTER TABLE package_revision DROP CONSTRAINT package_revision_license_id_fkey;" - change_license_id_type_on_package_table = "ALTER TABLE package ALTER COLUMN license_id TYPE text;" - change_license_id_type_on_package_revision_table = "ALTER TABLE package_revision ALTER COLUMN license_id TYPE text;" - drop_licenses_table = "DROP TABLE license CASCADE;" - - migrate_engine.execute(drop_fk_constraint_on_package_table) - migrate_engine.execute(drop_fk_constraint_on_package_revision_table) - migrate_engine.execute(change_license_id_type_on_package_table) - migrate_engine.execute(change_license_id_type_on_package_revision_table) - migrate_engine.execute(drop_licenses_table) - - # Set package license ids, and package revision license ids. - new_package_license_ids = _switch_package_license_ids( - old_package_license_ids, old_license_titles, map) - new_package_revision_license_ids = _switch_package_license_ids( - old_package_revision_license_ids, old_license_titles, map) - _set_new_package_license_ids(migrate_engine, new_package_license_ids) - _set_new_package_revision_license_ids(migrate_engine, new_package_revision_license_ids) - -def downgrade(migrate_engine): - raise NotImplementedError() - -def _check_map_has_old_license_titles(old_license_titles, map): - for title in old_license_titles.values(): - if title not in map: - raise Exception("The old license title '%s' wasn't found in the upgrade map. Decide which new license id should be substituted for this license and add an entry to the map (in ckan/migration/versions/018_adjust_licenses.py)." % title) - -def _get_old_license_titles(migrate_engine): - "Returns a dict of old license titles, keyed by old license id." - titles = {} - select_licenses = "SELECT id, name FROM license;" - q = migrate_engine.execute(select_licenses) - for id, title in q: - titles[id] = title - return titles - -def _get_old_package_license_ids(migrate_engine): - "Returns a dict of old license ids, keyed by package id." - old_ids = {} - select_licenses = "SELECT id, license_id FROM package;" - q = migrate_engine.execute(select_licenses) - for id, license_id in q: - old_ids[id] = license_id - return old_ids - -def _get_old_package_revision_license_ids(migrate_engine): - "Returns a dict of old license ids, keyed by package_revision id." - old_ids = {} - select_licenses = "SELECT id, license_id FROM package_revision;" - q = migrate_engine.execute(select_licenses) - for id, license_id in q: - old_ids[id] = license_id - return old_ids - -def _switch_package_license_ids(old_ids, old_license_titles, map): - "Returns a dict of new license ids, keyed by package id." - new_ids = {} - for (package_id, old_license_id) in old_ids.items(): - if old_license_id != None: - old_license_title = old_license_titles[old_license_id] - new_license_id = map[old_license_title] - new_ids[package_id] = new_license_id - print("Switched license_id %s to %s" % (old_license_id, new_license_id)) - return new_ids - -def _set_new_package_license_ids(migrate_engine, new_ids): - for (package_id, license_id) in new_ids.items(): - _set_package_license_id(migrate_engine, package_id, license_id) - -def _set_package_license_id(migrate_engine, package_id, license_id): - set_package_license_id = """UPDATE package SET license_id ='%s' where id = '%s';""" % (license_id, package_id) - migrate_engine.execute(set_package_license_id) - -def _set_new_package_revision_license_ids(migrate_engine, new_ids): - for (package_id, license_id) in new_ids.items(): - _set_package_revision_license_id(migrate_engine, package_id, license_id) - -def _set_package_revision_license_id(migrate_engine, package_id, license_id): - set_package_license_id = """UPDATE package_revision SET license_id ='%s' where id = '%s';""" % (license_id, package_id) - migrate_engine.execute(set_package_license_id) diff --git a/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py b/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py new file mode 100644 index 00000000000..9fd7f1dba87 --- /dev/null +++ b/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py @@ -0,0 +1,27 @@ +"""019 Pkg relationships_state + +Revision ID: b2eb6f34a638 +Revises: 05a0778051ca +Create Date: 2018-09-04 18:48:54.620241 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b2eb6f34a638' +down_revision = '05a0778051ca' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('package_relationship', sa.Column('state', sa.UnicodeText)) + op.add_column( + 'package_relationship_revision', sa.Column('state', sa.UnicodeText) + ) + + +def downgrade(): + op.drop_column('package_relationship', 'state') + op.drop_column('package_relationship_revision', 'state') diff --git a/ckan/migration/versions/019_pkg_relationships_state.py b/ckan/migration/versions/019_pkg_relationships_state.py deleted file mode 100644 index e70ad48a21c..00000000000 --- a/ckan/migration/versions/019_pkg_relationships_state.py +++ /dev/null @@ -1,26 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - package_relationship_table = Table('package_relationship', - metadata, autoload=True) - package_relationship_revision_table = Table('package_relationship_revision', - metadata, autoload=True) - - state_column = Column('state', UnicodeText) - state_column.create(package_relationship_table) - state_column = Column('state', UnicodeText) - state_column.create(package_relationship_revision_table) - # No package relationship objects exist to migrate, so no - # need to populate state column - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/020_69a0b0efc609_add_changeset.py b/ckan/migration/versions/020_69a0b0efc609_add_changeset.py new file mode 100644 index 00000000000..e8335313a23 --- /dev/null +++ b/ckan/migration/versions/020_69a0b0efc609_add_changeset.py @@ -0,0 +1,68 @@ +"""020 Add changeset + +Revision ID: 69a0b0efc609 +Revises: b2eb6f34a638 +Create Date: 2018-09-04 18:48:54.952113 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '69a0b0efc609' +down_revision = 'b2eb6f34a638' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'changeset', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('closes_id', sa.UnicodeText, nullable=True), + sa.Column('follows_id', sa.UnicodeText, nullable=True), + sa.Column('meta', sa.UnicodeText, nullable=True), + sa.Column('branch', sa.UnicodeText, nullable=True), + sa.Column( + 'timestamp', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + sa.Column('is_working', sa.Boolean, default=False), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + nullable=True + ), + sa.Column( + 'added_here', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + ) + + op.create_table( + 'change', + sa.Column('ref', sa.UnicodeText, nullable=True), + sa.Column('diff', sa.UnicodeText, nullable=True), + sa.Column( + 'changeset_id', sa.UnicodeText, sa.ForeignKey('changeset.id') + ), + ) + + op.create_table( + 'changemask', + sa.Column('ref', sa.UnicodeText, primary_key=True), + sa.Column( + 'timestamp', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + ) + + +def downgrade(): + op.drop_table('changeset') + op.drop_table('change') + op.drop_table('changemask') diff --git a/ckan/migration/versions/020_add_changeset.py b/ckan/migration/versions/020_add_changeset.py deleted file mode 100644 index 4c41a6c1d33..00000000000 --- a/ckan/migration/versions/020_add_changeset.py +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -from migrate.changeset.constraint import PrimaryKeyConstraint - - - -def upgrade(migrate_engine): - metadata = MetaData() - - revision_table = Table('revision', metadata, - Column('id', UnicodeText, primary_key=True), - ) - - changeset_table = Table('changeset', metadata, - Column('id', UnicodeText, primary_key=True), - Column('closes_id', UnicodeText, nullable=True), - Column('follows_id', UnicodeText, nullable=True), - Column('meta', UnicodeText, nullable=True), - Column('branch', UnicodeText, nullable=True), - Column('timestamp', DateTime, default=datetime.datetime.utcnow), - Column('is_working', Boolean, default=False), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), nullable=True), - Column('added_here', DateTime, default=datetime.datetime.utcnow), - ) - - change_table = Table('change', metadata, - Column('ref', UnicodeText, nullable=True), - Column('diff', UnicodeText, nullable=True), - Column('changeset_id', UnicodeText, ForeignKey('changeset.id')), - ) - - changemask_table = Table('changemask', metadata, - Column('ref', UnicodeText, primary_key=True), - Column('timestamp', DateTime, default=datetime.datetime.utcnow), - ) - metadata.bind = migrate_engine - changeset_table.create() - change_table.create() - changemask_table.create() - -def downgrade(migrate_engine): - change_table.drop() - changeset_table.drop() - diff --git a/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py b/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py new file mode 100644 index 00000000000..e5b468af436 --- /dev/null +++ b/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py @@ -0,0 +1,102 @@ +"""021 Postgresql upgrade.sql + +Revision ID: 765143af2ba3 +Revises: 4a8577e55a02 +Create Date: 2018-09-04 18:48:55.958481 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '765143af2ba3' +down_revision = '69a0b0efc609' +branch_labels = None +depends_on = None + +indexes = ( + ('idx_extra_pkg_id', 'package_extra', ['package_id']), + ('idx_extra_id_pkg_id', 'package_extra', ['id', 'package_id']), + ('idx_group_pkg_id', 'package_group', ['package_id']), + ('idx_extra_grp_id_pkg_id', 'package_group', ['group_id', 'package_id']), + ('idx_pkg_id', 'package', ['id']), + ('idx_pkg_name', 'package', ['name']), + ('idx_pkg_title', 'package', ['title']), + ('idx_pkg_lname', 'package', [sa.text('lower(name)')]), + ('idx_pkg_uname', 'package', [sa.text('upper(name)')]), + ('idx_pkg_rev_id', 'package', ['revision_id']), + ('idx_pkg_sid', 'package', ['id', 'state']), + ('idx_pkg_sname', 'package', ['name', 'state']), + ('idx_pkg_stitle', 'package', ['title', 'state']), + ('idx_pkg_slname', 'package', [sa.text('lower(name)'), 'state']), + ('idx_pkg_suname', 'package', [sa.text('upper(name)'), 'state']), + ('idx_pkg_srev_id', 'package', ['revision_id', 'state']), + ('idx_pkg_revision_id', 'package_revision', ['id']), + ('idx_pkg_revision_name', 'package_revision', ['name']), + ('idx_pkg_revision_rev_id', 'package_revision', ['revision_id']), + ('idx_rev_state', 'revision', ['state']), + ('idx_tag_id', 'tag', ['id']), + ('idx_tag_name', 'tag', ['name']), + ('idx_package_tag_id', 'package_tag', ['id']), + ('idx_package_tag_tag_id', 'package_tag', ['tag_id']), + ('idx_package_tag_pkg_id', 'package_tag', ['package_id']), + ('idx_package_tag_pkg_id_tag_id', 'package_tag', ['tag_id', 'package_id']), + ('idx_package_tag_revision_id', 'package_tag_revision', ['id']), + ('idx_package_tag_revision_tag_id', 'package_tag_revision', ['tag_id']), + ( + 'idx_package_tag_revision_rev_id', 'package_tag_revision', + ['revision_id'] + ), + ( + 'idx_package_tag_revision_pkg_id', 'package_tag_revision', + ['package_id'] + ), + ( + 'idx_package_tag_revision_pkg_id_tag_id', 'package_tag_revision', + ['tag_id', 'package_id'] + ), + ('idx_rating_id', 'rating', ['id']), + ('idx_rating_user_id', 'rating', ['user_id']), + ('idx_rating_package_id', 'rating', ['package_id']), + ('idx_user_id', 'user', ['id']), + ('idx_user_name', 'user', ['name']), + # ('idx_uor_id', 'user_object_role', ['id']), + # ('idx_uor_user_id', 'user_object_role', ['user_id']), + ('idx_uor_context', 'user_object_role', ['context']), + ('idx_uor_role', 'user_object_role', ['role']), + ('idx_uor_user_id_role', 'user_object_role', ['user_id', 'role']), + ('idx_ra_role', 'role_action', ['role']), + ('idx_ra_action', 'role_action', ['action']), + ('idx_ra_role_action', 'role_action', ['action', 'role']), + ('idx_group_id', 'group', ['id']), + ('idx_group_name', 'group', ['name']), + ('idx_package_group_id', 'package_group', ['id']), + ('idx_package_group_group_id', 'package_group', ['group_id']), + ('idx_package_group_pkg_id', 'package_group', ['package_id']), + ( + 'idx_package_group_pkg_id_group_id', 'package_group', + ['group_id', 'package_id'] + ), + ('idx_package_resource_id', 'package_resource', ['id']), + ('idx_package_resource_url', 'package_resource', ['url']), + ('idx_package_resource_pkg_id', 'package_resource', ['package_id']), + ( + 'idx_package_resource_pkg_id_resource_id', 'package_resource', + ['package_id', 'id'] + ), + ( + 'idx_package_resource_rev_id', 'package_resource_revision', + ['revision_id'] + ), + ('idx_package_extra_rev_id', 'package_extra_revision', ['revision_id']), +) + + +def upgrade(): + for name, table, columns in indexes: + op.create_index(name, table, columns) + + +def downgrade(): + for name, table, _ in indexes: + op.drop_index(name, table) diff --git a/ckan/migration/versions/021_postgres_downgrade.sql b/ckan/migration/versions/021_postgres_downgrade.sql deleted file mode 100644 index 332126148a4..00000000000 --- a/ckan/migration/versions/021_postgres_downgrade.sql +++ /dev/null @@ -1 +0,0 @@ -# NOP \ No newline at end of file diff --git a/ckan/migration/versions/021_postgres_upgrade.sql b/ckan/migration/versions/021_postgres_upgrade.sql deleted file mode 100644 index af97af7e462..00000000000 --- a/ckan/migration/versions/021_postgres_upgrade.sql +++ /dev/null @@ -1,73 +0,0 @@ -CREATE INDEX idx_extra_pkg_id ON package_extra (package_id); -CREATE INDEX idx_extra_id_pkg_id ON package_extra (id, package_id); - -CREATE INDEX idx_group_pkg_id ON package_group (package_id); -CREATE INDEX idx_extra_grp_id_pkg_id ON package_group (group_id, package_id); - -CREATE INDEX idx_pkg_id ON package (id); -CREATE INDEX idx_pkg_name ON package (name); -CREATE INDEX idx_pkg_title ON package (title); -CREATE INDEX idx_pkg_lname ON package (lower(name)); -CREATE INDEX idx_pkg_uname ON package (upper(name)); -CREATE INDEX idx_pkg_rev_id ON package (revision_id); - -CREATE INDEX idx_pkg_sid ON package (id,state); -CREATE INDEX idx_pkg_sname ON package (name,state); -CREATE INDEX idx_pkg_stitle ON package (title,state); -CREATE INDEX idx_pkg_slname ON package (lower(name),state); -CREATE INDEX idx_pkg_suname ON package (upper(name),state); -CREATE INDEX idx_pkg_srev_id ON package (revision_id,state); - -CREATE INDEX idx_pkg_revision_id ON package_revision (id); -CREATE INDEX idx_pkg_revision_name ON package_revision (name); -CREATE INDEX idx_pkg_revision_rev_id ON package_revision (revision_id); - -CREATE INDEX idx_rev_state ON revision (state); - -CREATE INDEX idx_tag_id ON tag (id); -CREATE INDEX idx_tag_name ON tag (name); - -CREATE INDEX idx_package_tag_id ON package_tag (id); -CREATE INDEX idx_package_tag_tag_id ON package_tag (tag_id); -CREATE INDEX idx_package_tag_pkg_id ON package_tag (package_id); -CREATE INDEX idx_package_tag_pkg_id_tag_id ON package_tag (tag_id, package_id); - -CREATE INDEX idx_package_tag_revision_id ON package_tag_revision (id); -CREATE INDEX idx_package_tag_revision_tag_id ON package_tag_revision (tag_id); -CREATE INDEX idx_package_tag_revision_rev_id ON package_tag_revision (revision_id); -CREATE INDEX idx_package_tag_revision_pkg_id ON package_tag_revision (package_id); -CREATE INDEX idx_package_tag_revision_pkg_id_tag_id ON package_tag_revision (tag_id, package_id); - -CREATE INDEX idx_rating_id ON rating (id); -CREATE INDEX idx_rating_user_id ON rating (user_id); -CREATE INDEX idx_rating_package_id ON rating (package_id); - -CREATE INDEX idx_user_id ON "user" (id); -CREATE INDEX idx_user_name ON "user" (name); - -CREATE INDEX idx_uor_id ON user_object_role (id); -CREATE INDEX idx_uor_user_id ON user_object_role (user_id); -CREATE INDEX idx_uor_context ON user_object_role (context); -CREATE INDEX idx_uor_role ON user_object_role (role); - -CREATE INDEX idx_uor_user_id_role ON user_object_role (user_id,role); -CREATE INDEX idx_ra_role ON role_action (role); -CREATE INDEX idx_ra_action ON role_action (action); -CREATE INDEX idx_ra_role_action ON role_action (action,role); - -CREATE INDEX idx_group_id ON "group" (id); -CREATE INDEX idx_group_name ON "group" (name); - -CREATE INDEX idx_package_group_id ON package_group (id); -CREATE INDEX idx_package_group_group_id ON package_group (group_id); -CREATE INDEX idx_package_group_pkg_id ON package_group (package_id); -CREATE INDEX idx_package_group_pkg_id_group_id ON package_group (group_id, package_id); - -CREATE INDEX idx_package_resource_id ON package_resource (id); -CREATE INDEX idx_package_resource_url ON package_resource (url); -CREATE INDEX idx_package_resource_pkg_id ON package_resource (package_id); -CREATE INDEX idx_package_resource_pkg_id_resource_id ON package_resource (package_id, id); - -CREATE INDEX idx_package_resource_rev_id ON package_resource_revision (revision_id); -CREATE INDEX idx_package_extra_rev_id ON package_extra_revision (revision_id); - diff --git a/ckan/migration/versions/021_postgresql_downgrade.sql b/ckan/migration/versions/021_postgresql_downgrade.sql deleted file mode 100644 index 332126148a4..00000000000 --- a/ckan/migration/versions/021_postgresql_downgrade.sql +++ /dev/null @@ -1 +0,0 @@ -# NOP \ No newline at end of file diff --git a/ckan/migration/versions/021_postgresql_upgrade.sql b/ckan/migration/versions/021_postgresql_upgrade.sql deleted file mode 100644 index af97af7e462..00000000000 --- a/ckan/migration/versions/021_postgresql_upgrade.sql +++ /dev/null @@ -1,73 +0,0 @@ -CREATE INDEX idx_extra_pkg_id ON package_extra (package_id); -CREATE INDEX idx_extra_id_pkg_id ON package_extra (id, package_id); - -CREATE INDEX idx_group_pkg_id ON package_group (package_id); -CREATE INDEX idx_extra_grp_id_pkg_id ON package_group (group_id, package_id); - -CREATE INDEX idx_pkg_id ON package (id); -CREATE INDEX idx_pkg_name ON package (name); -CREATE INDEX idx_pkg_title ON package (title); -CREATE INDEX idx_pkg_lname ON package (lower(name)); -CREATE INDEX idx_pkg_uname ON package (upper(name)); -CREATE INDEX idx_pkg_rev_id ON package (revision_id); - -CREATE INDEX idx_pkg_sid ON package (id,state); -CREATE INDEX idx_pkg_sname ON package (name,state); -CREATE INDEX idx_pkg_stitle ON package (title,state); -CREATE INDEX idx_pkg_slname ON package (lower(name),state); -CREATE INDEX idx_pkg_suname ON package (upper(name),state); -CREATE INDEX idx_pkg_srev_id ON package (revision_id,state); - -CREATE INDEX idx_pkg_revision_id ON package_revision (id); -CREATE INDEX idx_pkg_revision_name ON package_revision (name); -CREATE INDEX idx_pkg_revision_rev_id ON package_revision (revision_id); - -CREATE INDEX idx_rev_state ON revision (state); - -CREATE INDEX idx_tag_id ON tag (id); -CREATE INDEX idx_tag_name ON tag (name); - -CREATE INDEX idx_package_tag_id ON package_tag (id); -CREATE INDEX idx_package_tag_tag_id ON package_tag (tag_id); -CREATE INDEX idx_package_tag_pkg_id ON package_tag (package_id); -CREATE INDEX idx_package_tag_pkg_id_tag_id ON package_tag (tag_id, package_id); - -CREATE INDEX idx_package_tag_revision_id ON package_tag_revision (id); -CREATE INDEX idx_package_tag_revision_tag_id ON package_tag_revision (tag_id); -CREATE INDEX idx_package_tag_revision_rev_id ON package_tag_revision (revision_id); -CREATE INDEX idx_package_tag_revision_pkg_id ON package_tag_revision (package_id); -CREATE INDEX idx_package_tag_revision_pkg_id_tag_id ON package_tag_revision (tag_id, package_id); - -CREATE INDEX idx_rating_id ON rating (id); -CREATE INDEX idx_rating_user_id ON rating (user_id); -CREATE INDEX idx_rating_package_id ON rating (package_id); - -CREATE INDEX idx_user_id ON "user" (id); -CREATE INDEX idx_user_name ON "user" (name); - -CREATE INDEX idx_uor_id ON user_object_role (id); -CREATE INDEX idx_uor_user_id ON user_object_role (user_id); -CREATE INDEX idx_uor_context ON user_object_role (context); -CREATE INDEX idx_uor_role ON user_object_role (role); - -CREATE INDEX idx_uor_user_id_role ON user_object_role (user_id,role); -CREATE INDEX idx_ra_role ON role_action (role); -CREATE INDEX idx_ra_action ON role_action (action); -CREATE INDEX idx_ra_role_action ON role_action (action,role); - -CREATE INDEX idx_group_id ON "group" (id); -CREATE INDEX idx_group_name ON "group" (name); - -CREATE INDEX idx_package_group_id ON package_group (id); -CREATE INDEX idx_package_group_group_id ON package_group (group_id); -CREATE INDEX idx_package_group_pkg_id ON package_group (package_id); -CREATE INDEX idx_package_group_pkg_id_group_id ON package_group (group_id, package_id); - -CREATE INDEX idx_package_resource_id ON package_resource (id); -CREATE INDEX idx_package_resource_url ON package_resource (url); -CREATE INDEX idx_package_resource_pkg_id ON package_resource (package_id); -CREATE INDEX idx_package_resource_pkg_id_resource_id ON package_resource (package_id, id); - -CREATE INDEX idx_package_resource_rev_id ON package_resource_revision (revision_id); -CREATE INDEX idx_package_extra_rev_id ON package_extra_revision (revision_id); - diff --git a/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py b/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py new file mode 100644 index 00000000000..d7230ca9000 --- /dev/null +++ b/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py @@ -0,0 +1,29 @@ +"""022 Add group_extras + +Revision ID: 7b324ca6c0dc +Revises: c7743043ed99 +Create Date: 2018-09-04 18:48:56.635671 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '7b324ca6c0dc' +down_revision = '765143af2ba3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'group_extra', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + sa.Column('key', sa.UnicodeText), + sa.Column('value', sa.UnicodeText), + ) + + +def downgrade(): + op.drop_table('group_extra') diff --git a/ckan/migration/versions/022_add_group_extras.py b/ckan/migration/versions/022_add_group_extras.py deleted file mode 100644 index 06db6479a8a..00000000000 --- a/ckan/migration/versions/022_add_group_extras.py +++ /dev/null @@ -1,53 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset -import vdm.sqlalchemy -import uuid -from sqlalchemy import types - -from ckan.common import json - -class JsonType(types.TypeDecorator): - '''Store data as JSON serializing on save and unserializing on use. - ''' - impl = types.UnicodeText - - def process_bind_param(self, value, engine): - if value is None or value == {}: # ensure we stores nulls in db not json "null" - return None - else: - # ensure_ascii=False => allow unicode but still need to convert - return unicode(json.dumps(value, ensure_ascii=False)) - - def process_result_value(self, value, engine): - if value is None: - return None - else: - return json.loads(value) - - def copy(self): - return JsonType(self.impl.length) - -def make_uuid(): - return unicode(uuid.uuid4()) - - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - group_table = Table('group', metadata, autoload=True) - group_extra_table = Table('group_extra', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('group_id', UnicodeText, ForeignKey('group.id')), - Column('key', UnicodeText), - Column('value', JsonType), - ) - - group_extra_table.create() - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py b/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py new file mode 100644 index 00000000000..e7cb28f1afe --- /dev/null +++ b/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py @@ -0,0 +1,49 @@ +"""023 Add harvesting + +Revision ID: 87fdd05f0744 +Revises: 7b324ca6c0dc +Create Date: 2018-09-04 18:48:56.975156 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '87fdd05f0744' +down_revision = '7b324ca6c0dc' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'harvest_source', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('status', sa.UnicodeText, server_default=u'New'), + sa.Column('url', sa.UnicodeText, unique=True, nullable=False), + sa.Column('description', sa.UnicodeText, default=u''), + sa.Column('user_ref', sa.UnicodeText, default=u''), + sa.Column('publisher_ref', sa.UnicodeText, default=u''), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + ) + + op.create_table( + 'harvesting_job', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('status', sa.UnicodeText, default=u'', nullable=False), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + sa.Column('user_ref', sa.UnicodeText, nullable=False), + sa.Column('report', sa.UnicodeText, default=u''), + sa.Column( + 'source_id', sa.UnicodeText, sa.ForeignKey('harvest_source.id') + ), + ) + + +def downgrade(): + op.drop_table('harvesting_job') + op.drop_table('harvest_source') diff --git a/ckan/migration/versions/023_add_harvesting.py b/ckan/migration/versions/023_add_harvesting.py deleted file mode 100644 index b110285cfcc..00000000000 --- a/ckan/migration/versions/023_add_harvesting.py +++ /dev/null @@ -1,42 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -import uuid -from migrate.changeset.constraint import PrimaryKeyConstraint - - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - harvest_source_table = Table('harvest_source', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('status', UnicodeText, default=u'New'), - Column('url', UnicodeText, unique=True, nullable=False), - Column('description', UnicodeText, default=u''), - Column('user_ref', UnicodeText, default=u''), - Column('publisher_ref', UnicodeText, default=u''), - Column('created', DateTime, default=datetime.datetime.utcnow), - ) - - harvesting_job_table = Table('harvesting_job', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('status', UnicodeText, default=u'', nullable=False), - Column('created', DateTime, default=datetime.datetime.utcnow), - Column('user_ref', UnicodeText, nullable=False), - Column('report', UnicodeText, default=u''), - Column('source_id', UnicodeText, ForeignKey('harvest_source.id')), - ) - - metadata.bind = migrate_engine - harvest_source_table.create(checkfirst=True) - harvesting_job_table.create(checkfirst=True) - -def downgrade(migrate_engine): - metadata.bind = migrate_engine - harvesting_job_table.drop() - harvest_source_table.drop() - diff --git a/ckan/migration/versions/024_12981fe12484_add_harvested_document.py b/ckan/migration/versions/024_12981fe12484_add_harvested_document.py new file mode 100644 index 00000000000..a4ecc117e9f --- /dev/null +++ b/ckan/migration/versions/024_12981fe12484_add_harvested_document.py @@ -0,0 +1,29 @@ +"""024 Add harvested_document + +Revision ID: 12981fe12484 +Revises: 87fdd05f0744 +Create Date: 2018-09-04 18:48:57.309349 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '12981fe12484' +down_revision = '87fdd05f0744' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'harvested_document', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('created', sa.DateTime), + sa.Column('url', sa.UnicodeText, nullable=False), + sa.Column('content', sa.UnicodeText, nullable=False), + ) + + +def downgrade(): + op.drop_table('harvested_document') diff --git a/ckan/migration/versions/024_add_harvested_document.py b/ckan/migration/versions/024_add_harvested_document.py deleted file mode 100644 index 558e98cb6df..00000000000 --- a/ckan/migration/versions/024_add_harvested_document.py +++ /dev/null @@ -1,22 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime - - -def upgrade(migrate_engine): - metadata = MetaData() - harvested_document_table = Table('harvested_document', metadata, - Column('id', UnicodeText, primary_key=True), - Column('created', DateTime), - Column('url', UnicodeText, nullable=False), - Column('content', UnicodeText, nullable=False), - ) - metadata.bind = migrate_engine - harvested_document_table.create() - -def downgrade(migrate_engine): - metadata.bind = migrate_engine - harvested_document_table.drop() - diff --git a/ckan/migration/versions/025_add_authorization_groups.py b/ckan/migration/versions/025_add_authorization_groups.py deleted file mode 100644 index 75c74e672e8..00000000000 --- a/ckan/migration/versions/025_add_authorization_groups.py +++ /dev/null @@ -1,58 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -from datetime import datetime -import migrate.changeset -import vdm.sqlalchemy -import uuid -from sqlalchemy import types - -def make_uuid(): - return unicode(uuid.uuid4()) - - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - user_table = Table('user', metadata, autoload=True) - - authorization_group_table = Table('authorization_group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText), - Column('created', DateTime, default=datetime.now), - ) - - authorization_group_user_table = Table('authorization_group_user', metadata, - Column('authorization_group_id', UnicodeText, ForeignKey('authorization_group.id'), nullable=False), - Column('user_id', UnicodeText, ForeignKey('user.id'), nullable=False) - ) - - # make user nullable: - user_object_role_table = Table('user_object_role', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('user_id', UnicodeText, ForeignKey('user.id'), nullable=True), - Column('context', UnicodeText, nullable=False), - Column('role', UnicodeText) - ) - - authorization_group_role_table = Table('authorization_group_role', metadata, - Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True), - Column('authorization_group_id', UnicodeText, ForeignKey('authorization_group.id')), - ) - - authorization_group_table.create() - authorization_group_user_table.create() - authorization_group_role_table.create() - authorization_group_id = Column('authorized_group_id', UnicodeText, - ForeignKey('authorization_group.id'), nullable=True) - authorization_group_id.create(user_object_role_table) - -def downgrade(migrate_engine): - raise NotImplementedError() - - - - diff --git a/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py b/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py new file mode 100644 index 00000000000..0c78a31c4f7 --- /dev/null +++ b/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py @@ -0,0 +1,76 @@ +"""025 Add authorization groups + +Revision ID: b581622ad327 +Revises: 12981fe12484 +Create Date: 2018-09-04 18:48:57.649187 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b581622ad327' +down_revision = '12981fe12484' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'authorization_group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('name', sa.UnicodeText), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + ) + + op.create_table( + 'authorization_group_user', + sa.Column( + 'authorization_group_id', + sa.UnicodeText, + sa.ForeignKey('authorization_group.id'), + nullable=False + ), + sa.Column( + 'user_id', + sa.UnicodeText, + sa.ForeignKey('user.id'), + nullable=False + ) + ) + # make user nullable: + op.alter_column('user_object_role', 'user_id', nullable=True) + + op.create_table( + 'authorization_group_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column( + 'authorization_group_id', sa.UnicodeText, + sa.ForeignKey('authorization_group.id') + ), + ) + + op.add_column( + 'user_object_role', + sa.Column( + 'authorized_group_id', + sa.UnicodeText, + sa.ForeignKey('authorization_group.id'), + nullable=True + ) + ) + + +def downgrade(): + op.drop_column('user_object_role', 'authorized_group_id') + op.drop_table('authorization_group_role') + op.alter_column('user_object_role', 'user_id', nullable=False) + op.drop_table('authorization_group_user') + op.drop_table('authorization_group') diff --git a/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py b/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py new file mode 100644 index 00000000000..a15ceecfcdd --- /dev/null +++ b/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py @@ -0,0 +1,23 @@ +"""026 Authorization group user pk + +Revision ID: 3615b25af443 +Revises: b581622ad327 +Create Date: 2018-09-04 18:48:57.988110 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '3615b25af443' +down_revision = 'b581622ad327' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/026_authorization_group_user_pk.py b/ckan/migration/versions/026_authorization_group_user_pk.py deleted file mode 100644 index 8e788a27c7f..00000000000 --- a/ckan/migration/versions/026_authorization_group_user_pk.py +++ /dev/null @@ -1,37 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -from datetime import datetime -import migrate.changeset -import vdm.sqlalchemy -import uuid -from sqlalchemy import types - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - user_table = Table('user', metadata, autoload=True) - - authorization_group_table = Table('authorization_group', metadata, autoload=True) - - authorization_group_role_table = Table('authorization_group_role', metadata, - Column('user_object_role_id', UnicodeText, ForeignKey('user_object_role.id'), primary_key=True), - Column('authorization_group_id', UnicodeText, ForeignKey('authorization_group.id')), - ) -## id = Column('id', UnicodeText, primary_key=True, default=make_uuid) -## id.create(table=authorization_group_role_table, -## primary_key_name='blum' -### unique_name='id' -## ) - -def downgrade(migrate_engine): - raise NotImplementedError() - - - - diff --git a/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py b/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py new file mode 100644 index 00000000000..8c445bc851d --- /dev/null +++ b/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py @@ -0,0 +1,39 @@ +"""027 Adjust harvester + +Revision ID: 11e5745c6fc9 +Revises: 3615b25af443 +Create Date: 2018-09-04 18:48:58.333396 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '11e5745c6fc9' +down_revision = '3615b25af443' +branch_labels = None +depends_on = None + +table = 'harvested_document' + + +def upgrade(): + op.add_column(table, sa.Column('guid', sa.UnicodeText, server_default=u'')) + op.add_column( + table, + sa.Column( + 'source_id', sa.UnicodeText, sa.ForeignKey('harvest_source.id') + ) + ) + op.add_column( + table, + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')) + ) + op.drop_column(table, 'url') + + +def downgrade(): + op.drop_column(table, 'package_id') + op.drop_column(table, 'source_id') + op.drop_column(table, 'guid') + op.add_column(table, sa.Column('url', sa.UnicodeText, nullable=False)) diff --git a/ckan/migration/versions/027_adjust_harvester.py b/ckan/migration/versions/027_adjust_harvester.py deleted file mode 100644 index 4b4a6be39eb..00000000000 --- a/ckan/migration/versions/027_adjust_harvester.py +++ /dev/null @@ -1,34 +0,0 @@ -# encoding: utf-8 - -import warnings - -from sqlalchemy import exc as sa_exc -from sqlalchemy import * -from migrate import * -import migrate.changeset - -def upgrade(migrate_engine): - # ignore reflection warnings - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=sa_exc.SAWarning) - metadata = MetaData() - metadata.bind = migrate_engine - - harvest_source_table = Table('harvest_source', metadata, autoload=True) - package_table = Table('package', metadata, autoload=True) - - harvested_document_table = Table('harvested_document', metadata, - Column('url', UnicodeText, nullable=False), - Column('guid', UnicodeText, default=u''), - Column('source_id', UnicodeText, ForeignKey('harvest_source.id')), - Column('package_id', UnicodeText, ForeignKey('package.id')), - ) - - harvested_document_table.c.url.drop() - harvested_document_table.c.guid.create(harvested_document_table) - harvested_document_table.c.source_id.create(harvested_document_table) - harvested_document_table.c.package_id.create(harvested_document_table) - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py b/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py new file mode 100644 index 00000000000..f41b7d2af9c --- /dev/null +++ b/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py @@ -0,0 +1,23 @@ +"""028 Drop harvest_source_status + +Revision ID: cdd68fe9ba21 +Revises: 11e5745c6fc9 +Create Date: 2018-09-04 18:48:58.674039 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'cdd68fe9ba21' +down_revision = '11e5745c6fc9' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column('harvest_source', 'status', nullable=False) + + +def downgrade(): + op.alter_column('harvest_source', 'status', nullable=True) diff --git a/ckan/migration/versions/028_drop_harvest_source_status.py b/ckan/migration/versions/028_drop_harvest_source_status.py deleted file mode 100644 index 0f906a92b27..00000000000 --- a/ckan/migration/versions/028_drop_harvest_source_status.py +++ /dev/null @@ -1,18 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - - -def upgrade(migrate_engine): - metadata = MetaData() - harvested_source_table = Table('harvest_source', metadata, - Column('status', UnicodeText, nullable=False), - ) - metadata.bind = migrate_engine - harvested_source_table.c.status.drop() - -def downgrade(): - raise NotImplementedError() - diff --git a/ckan/migration/versions/029_1bfdf4240915_version_groups.py b/ckan/migration/versions/029_1bfdf4240915_version_groups.py new file mode 100644 index 00000000000..e7a72103aaa --- /dev/null +++ b/ckan/migration/versions/029_1bfdf4240915_version_groups.py @@ -0,0 +1,106 @@ +"""029 Version groups + +Revision ID: 1bfdf4240915 +Revises: cdd68fe9ba21 +Create Date: 2018-09-04 18:48:59.007126 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '1bfdf4240915' +down_revision = 'cdd68fe9ba21' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'group_revision', sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('name', sa.UnicodeText, nullable=False), + sa.Column('title', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), sa.Column('state', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column('continuity_id', sa.UnicodeText, sa.ForeignKey('group.id')) + ) + + op.create_table( + 'package_group_revision', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + sa.Column('state', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.UnicodeText, sa.ForeignKey('package_group.id') + ) + ) + + op.create_table( + 'group_extra_revision', + sa.Column( + 'id', + sa.UnicodeText, + primary_key=True, + ), sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + sa.Column('key', sa.UnicodeText), sa.Column('value', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.UnicodeText, sa.ForeignKey('group_extra.id') + ) + ) + + op.add_column('group', sa.Column('state', sa.UnicodeText)) + op.add_column('group', sa.Column('revision_id', sa.UnicodeText)) + op.create_foreign_key( + 'group_revision_id_fkey', 'group', 'revision', ['revision_id'], ['id'] + ) + + op.add_column('package_group', sa.Column('state', sa.UnicodeText)) + op.add_column('package_group', sa.Column('revision_id', sa.UnicodeText)) + op.create_foreign_key( + 'package_group_revision_id_fkey', 'package_group', 'revision', + ['revision_id'], ['id'] + ) + + op.add_column('group_extra', sa.Column('state', sa.UnicodeText)) + op.add_column('group_extra', sa.Column('revision_id', sa.UnicodeText)) + op.create_foreign_key( + 'group_extra_revision_id_fkey', 'group_extra', 'revision', + ['revision_id'], ['id'] + ) + + +def downgrade(): + op.drop_column('group_extra', 'revision_id') + op.drop_column('group_extra', 'state') + + op.drop_column('package_group', 'revision_id') + op.drop_column('package_group', 'state') + + op.drop_column('group', 'revision_id') + op.drop_column('group', 'state') + + op.drop_table('group_extra_revision') + op.drop_table('package_group_revision') + op.drop_table('group_revision') diff --git a/ckan/migration/versions/029_version_groups.py b/ckan/migration/versions/029_version_groups.py deleted file mode 100644 index 4c930aa97f6..00000000000 --- a/ckan/migration/versions/029_version_groups.py +++ /dev/null @@ -1,174 +0,0 @@ -# encoding: utf-8 - -import uuid - -from sqlalchemy import * -from sqlalchemy import types -from migrate import * -from datetime import datetime -import migrate.changeset -from migrate.changeset.constraint import ForeignKeyConstraint - -from ckan.common import json - -class JsonType(types.TypeDecorator): - '''Store data as JSON serializing on save and unserializing on use. - ''' - impl = types.UnicodeText - - def process_bind_param(self, value, engine): - if value is None or value == {}: # ensure we stores nulls in db not json "null" - return None - else: - # ensure_ascii=False => allow unicode but still need to convert - return unicode(json.dumps(value, ensure_ascii=False)) - - def process_result_value(self, value, engine): - if value is None: - return None - else: - return json.loads(value) - - def copy(self): - return JsonType(self.impl.length) - -def make_uuid(): - return unicode(uuid.uuid4()) - - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - - group_table = Table('group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText, unique=True, nullable=False), - Column('title', UnicodeText), - Column('description', UnicodeText), - Column('created', DateTime, default=datetime.now), - ) - - group_revision_table = Table('group_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText, nullable=False), - Column('title', UnicodeText), - Column('description', UnicodeText), - Column('created', DateTime, default=datetime.now), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('group.id')) - ) - - package_group_table = Table('package_group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText, ForeignKey('package.id')), - Column('group_id', UnicodeText, ForeignKey('group.id')), - ) - - package_group_revision_table = Table('package_group_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText, ForeignKey('package.id')), - Column('group_id', UnicodeText, ForeignKey('group.id')), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('package_group.id')) - ) - - group_extra_table = Table('group_extra', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('group_id', UnicodeText, ForeignKey('group.id')), - Column('key', UnicodeText), - Column('value', JsonType), - ) - - group_extra_revision_table = Table('group_extra_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('group_id', UnicodeText, ForeignKey('group.id')), - Column('key', UnicodeText), - Column('value', JsonType), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('group_extra.id')) - ) - - revision_table = Table('revision', metadata, autoload=True) - package_table = Table('package', metadata, autoload=True) - - rev_id = make_uuid() - q = revision_table.insert(values={'id': rev_id, - 'author': u'system', - 'message': u"Add versioning to groups, group_extras and package_groups", - 'timestamp': datetime.utcnow(), - 'state': u'active'}) - r = migrate_engine.execute(q) - - # handle groups: - - # BUG in sqlalchemy-migrate 0.4/0.5.4: "group" isn't escaped properly when sent to - # postgres. - # cf http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=32 - - #state = Column('state', UnicodeText) - #revision_id = Column('revision_id', UnicodeText) - #state.create(group_table) - #revision_id.create(group_table) - migrate_engine.execute('ALTER TABLE "group" ADD COLUMN state TEXT') - migrate_engine.execute('ALTER TABLE "group" ADD COLUMN revision_id TEXT') - #q = group_table.update(values={'state': 'active', - # 'revision_id': rev_id}) - #migrate_engine.execute(q) - migrate_engine.execute('UPDATE "group" SET revision_id = \'%s\', state=\'active\'' % rev_id) - #fk = ForeignKeyConstraint(['revision_id'], [revision_table.c.id], table=group_table) - #fk.create(migrate_engine) - migrate_engine.execute('ALTER TABLE "group" ADD CONSTRAINT "group_revision_id_fkey" ' + \ - 'FOREIGN KEY (revision_id) REFERENCES revision(id)') - - group_revision_table.create() - for row in migrate_engine.execute(group_table.select()): - group_rev = dict(row.items()) - group_rev['continuity_id'] = group_rev['id'] - - # otherwise, this doesn't get mapped due to the bug above: - group_rev['state'] = u'active' - group_rev['revision_id'] = rev_id - - q = group_revision_table.insert(values=group_rev) - migrate_engine.execute(q) - - - state = Column('state', UnicodeText) - revision_id = Column('revision_id', UnicodeText) - state.create(package_group_table) - revision_id.create(package_group_table) - q = package_group_table.update(values={'state': u'active', - 'revision_id': rev_id}) - migrate_engine.execute(q) - fk = ForeignKeyConstraint(['revision_id'], [revision_table.c.id], table=package_group_table, name = 'package_group_revision_id_fkey') - fk.create(migrate_engine) - package_group_revision_table.create() - for row in migrate_engine.execute(package_group_table.select()): - pkg_group_rev = dict(row.items()) - pkg_group_rev['continuity_id'] = pkg_group_rev['id'] - q = package_group_revision_table.insert(values=pkg_group_rev) - migrate_engine.execute(q) - - state = Column('state', UnicodeText) - revision_id = Column('revision_id', UnicodeText) - state.create(group_extra_table) - revision_id.create(group_extra_table) - q = group_extra_table.update(values={'state': u'active', - 'revision_id': rev_id}) - migrate_engine.execute(q) - fk = ForeignKeyConstraint(['revision_id'], [revision_table.c.id], table=group_extra_table, name='group_extra_revision_id_fkey') - fk.create(migrate_engine) - group_extra_revision_table.create() - for row in migrate_engine.execute(group_extra_table.select()): - group_extra_rev = dict(row.items()) - group_extra_rev['continuity_id'] = group_rev['id'] - q = group_extra_revision_table.insert(values=group_extra_rev) - migrate_engine.execute(q) - - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/030_additional_user_attributes.py b/ckan/migration/versions/030_additional_user_attributes.py deleted file mode 100644 index 84808afbbae..00000000000 --- a/ckan/migration/versions/030_additional_user_attributes.py +++ /dev/null @@ -1,24 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from sqlalchemy import types -from migrate import * -from datetime import datetime -import migrate.changeset -import uuid - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - user_sql = 'ALTER TABLE "user" ADD openid TEXT' - migrate_engine.execute(user_sql) - user_sql = 'ALTER TABLE "user" ADD password TEXT' - migrate_engine.execute(user_sql) - user_sql = 'ALTER TABLE "user" ADD fullname TEXT' - migrate_engine.execute(user_sql) - user_sql = 'ALTER TABLE "user" ADD email TEXT' - migrate_engine.execute(user_sql) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py b/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py new file mode 100644 index 00000000000..d918934c70d --- /dev/null +++ b/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py @@ -0,0 +1,29 @@ +"""030 Additional user_attributes + +Revision ID: b16cbf164c8a +Revises: 1bfdf4240915 +Create Date: 2018-09-04 18:48:59.340276 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b16cbf164c8a' +down_revision = '1bfdf4240915' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('user', sa.Column('openid', sa.UnicodeText)) + op.add_column('user', sa.Column('password', sa.UnicodeText)) + op.add_column('user', sa.Column('fullname', sa.UnicodeText)) + op.add_column('user', sa.Column('email', sa.UnicodeText)) + + +def downgrade(): + op.drop_column('user', 'openid') + op.drop_column('user', 'password') + op.drop_column('user', 'fullname') + op.drop_column('user', 'email') diff --git a/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py b/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py new file mode 100644 index 00000000000..ad5d579dafa --- /dev/null +++ b/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py @@ -0,0 +1,23 @@ +"""031 Move openid to new_field + +Revision ID: 1b05245167d6 +Revises: b16cbf164c8a +Create Date: 2018-09-04 18:48:59.666938 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '1b05245167d6' +down_revision = 'b16cbf164c8a' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/031_move_openid_to_new_field.py b/ckan/migration/versions/031_move_openid_to_new_field.py deleted file mode 100644 index 9a45cc51241..00000000000 --- a/ckan/migration/versions/031_move_openid_to_new_field.py +++ /dev/null @@ -1,39 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from sqlalchemy import types -from migrate import * -from datetime import datetime -import migrate.changeset -import uuid - - -def make_uuid(): - return unicode(uuid.uuid4()) - - -def upgrade(migrate_engine): - metadata = MetaData() - user_table = Table('user', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('name', UnicodeText), - Column('openid', UnicodeText), - Column('password', UnicodeText), - Column('fullname', UnicodeText), - Column('email', UnicodeText), - Column('apikey', UnicodeText, default=make_uuid), - Column('created', DateTime, default=datetime.now), - Column('about', UnicodeText), - ) - metadata.bind = migrate_engine - for row in migrate_engine.execute(user_table.select()): - user = dict(row.items()) - name = user.get('name').lower().strip() - if name.startswith('http://') or name.startswith('https://'): - user['openid'] = name - q = user_table.update(user_table.c.id==user.get('id'), - values=user) - migrate_engine.execute(q) - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/032_add_extra_info_field_to_resources.py b/ckan/migration/versions/032_add_extra_info_field_to_resources.py deleted file mode 100644 index 46ab4b98ff7..00000000000 --- a/ckan/migration/versions/032_add_extra_info_field_to_resources.py +++ /dev/null @@ -1,20 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from sqlalchemy import types -from migrate import * -from datetime import datetime -import migrate.changeset -import uuid - - -def upgrade(migrate_engine): - metadata = MetaData(migrate_engine) - user_sql = 'ALTER TABLE package_resource ADD COLUMN extras text' - migrate_engine.execute(user_sql) - user_sql = 'ALTER TABLE package_resource_revision ADD COLUMN extras text' - migrate_engine.execute(user_sql) - -def downgrade(): - raise NotImplementedError() - diff --git a/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py b/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py new file mode 100644 index 00000000000..7ad8607049e --- /dev/null +++ b/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py @@ -0,0 +1,27 @@ +"""032 Add extra info field_to_resources + +Revision ID: d89e0731422d +Revises: 1b05245167d6 +Create Date: 2018-09-04 18:49:00.003141 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'd89e0731422d' +down_revision = '1b05245167d6' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('package_resource', sa.Column('extras', sa.UnicodeText)) + op.add_column( + 'package_resource_revision', sa.Column('extras', sa.UnicodeText) + ) + + +def downgrade(): + op.drop_column('package_resource', 'extras') + op.drop_column('package_resource_revision', 'extras') diff --git a/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py b/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py new file mode 100644 index 00000000000..a2feaa1037f --- /dev/null +++ b/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py @@ -0,0 +1,26 @@ +"""033 Auth group user id_add_conditional + +Revision ID: 6da92ef2df15 +Revises: d89e0731422d +Create Date: 2018-09-04 18:49:00.347621 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6da92ef2df15' +down_revision = 'd89e0731422d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('authorization_group_user', sa.Column('id', sa.UnicodeText)) + op.create_primary_key( + 'authorization_group_user_pkey', 'authorization_group_user', ['id'] + ) + + +def downgrade(): + op.drop_column('authorization_group_user', 'id') diff --git a/ckan/migration/versions/033_auth_group_user_id_add_conditional.py b/ckan/migration/versions/033_auth_group_user_id_add_conditional.py deleted file mode 100644 index dbe5fa6e1e6..00000000000 --- a/ckan/migration/versions/033_auth_group_user_id_add_conditional.py +++ /dev/null @@ -1,32 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -import uuid -from migrate.changeset.constraint import PrimaryKeyConstraint - - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - authorization_group_user_table = Table('authorization_group_user', - metadata, autoload=True) - - try: - ##check if id column already exists - authorization_group_user_table.c["id"] - return - except KeyError: - pass - - - id_col = Column('id', UnicodeText, primary_key=True, default=make_uuid) - id_col.create(authorization_group_user_table, - primary_key_name='authorization_group_user_pkey') - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/034_6c600693af5b_resource_group_table.py b/ckan/migration/versions/034_6c600693af5b_resource_group_table.py new file mode 100644 index 00000000000..8696b41193d --- /dev/null +++ b/ckan/migration/versions/034_6c600693af5b_resource_group_table.py @@ -0,0 +1,158 @@ +"""034 Resource group table + +Revision ID: 6c600693af5b +Revises: 6da92ef2df15 +Create Date: 2018-09-04 18:49:00.683101 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6c600693af5b' +down_revision = '6da92ef2df15' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'resource_group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + sa.Column('label', sa.UnicodeText), + sa.Column('sort_order', sa.UnicodeText), + sa.Column('extras', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, sa.ForeignKey('revision.id')), + ) + + op.create_table( + 'resource_group_revision', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + sa.Column('label', sa.UnicodeText), + sa.Column('sort_order', sa.UnicodeText), + sa.Column('extras', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.UnicodeText, + sa.ForeignKey('resource_group.id') + ) + ) + op.alter_column( + 'package_resource', 'package_id', new_column_name='resource_group_id' + ) + op.alter_column( + 'package_resource_revision', + 'package_id', + new_column_name='resource_group_id' + ) + + op.rename_table('package_resource', 'resource') + op.rename_table('package_resource_revision', 'resource_revision') + op.execute('ALTER INDEX package_resource_pkey RENAME TO resource_pkey') + op.execute( + 'ALTER INDEX package_resource_revision_pkey RENAME TO resource_revision_pkey' + ) + + op.drop_constraint( + 'package_resource_revision_continuity_id_fkey', 'resource_revision' + ) + op.drop_constraint( + 'package_resource_revision_package_id_fkey', 'resource_revision' + ) + op.drop_constraint( + 'package_resource_revision_revision_id_fkey', 'resource_revision' + ) + op.drop_constraint('package_resource_revision_id_fkey', 'resource') + op.drop_constraint('package_resource_package_id_fkey', 'resource') + + op.create_foreign_key( + 'resource_resource_group_id_fkey', 'resource', 'resource_group', + ['resource_group_id'], ['id'] + ) + op.create_foreign_key( + 'resource_revision_id_fkey', 'resource', 'revision', ['revision_id'], + ['id'] + ) + op.create_foreign_key( + 'resource_revision_continuity_id_fkey', 'resource_revision', + 'resource', ['continuity_id'], ['id'] + ) + op.create_foreign_key( + 'resource_revision_resource_group_id_fkey', 'resource_revision', + 'resource_group', ['resource_group_id'], ['id'] + ) + op.create_foreign_key( + 'resource_revision_revision_id_fkey', 'resource_revision', 'revision', + ['revision_id'], ['id'] + ) + + +def downgrade(): + op.drop_constraint('resource_resource_group_id_fkey', 'resource') + op.drop_constraint('resource_revision_id_fkey', 'resource') + op.drop_constraint( + 'resource_revision_continuity_id_fkey', 'resource_revision' + ) + op.drop_constraint( + 'resource_revision_resource_group_id_fkey', 'resource_revision' + ) + op.drop_constraint( + 'resource_revision_revision_id_fkey', 'resource_revision' + ) + + op.rename_table('resource_revision', 'package_resource_revision') + op.rename_table('resource', 'package_resource') + op.execute('ALTER INDEX resource_pkey RENAME TO package_resource_pkey') + op.execute( + 'ALTER INDEX resource_revision_pkey RENAME TO package_resource_revision_pkey' + ) + + op.alter_column( + 'package_resource_revision', + 'resource_group_id', + new_column_name='package_id' + ) + op.alter_column( + 'package_resource', 'resource_group_id', new_column_name='package_id' + ) + + op.create_foreign_key( + 'package_resource_package_id_fkey', 'package_resource', 'package', + ['package_id'], ['id'] + ) + op.create_foreign_key( + 'package_resource_revision_id_fkey', 'package_resource', 'revision', + ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'package_resource_revision_revision_id_fkey', + 'package_resource_revision', 'revision', ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'package_resource_revision_package_id_fkey', + 'package_resource_revision', 'package', ['package_id'], ['id'] + ) + op.create_foreign_key( + 'package_resource_revision_continuity_id_fkey', + 'package_resource_revision', 'package_resource', ['continuity_id'], + ['id'] + ) + op.create_index( + 'idx_package_resource_pkg_id', 'package_resource', ['package_id'] + ) + op.create_index( + 'idx_package_resource_pkg_id_resource_id', 'package_resource', + ['package_id', 'id'] + ) + + op.drop_table('resource_group_revision') + op.drop_table('resource_group') diff --git a/ckan/migration/versions/034_resource_group_table.py b/ckan/migration/versions/034_resource_group_table.py deleted file mode 100644 index 5e38e5db641..00000000000 --- a/ckan/migration/versions/034_resource_group_table.py +++ /dev/null @@ -1,209 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -import uuid -import migrate.changeset -from migrate.changeset.constraint import PrimaryKeyConstraint -from ckan.model.types import JsonDictType - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - - metadata = MetaData(migrate_engine) - - package = Table('package', metadata, autoload=True) - - resource = Table( - 'package_resource', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText, ForeignKey('package.id')), - Column('url', UnicodeText, nullable=False), - Column('format', UnicodeText), - Column('description', UnicodeText), - Column('hash', UnicodeText), - Column('position', Integer), - Column('extras', JsonDictType), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - ) - - resource_revision = Table( - 'package_resource_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText), - Column('url', UnicodeText, nullable=False), - Column('format', UnicodeText), - Column('description', UnicodeText), - Column('hash', UnicodeText), - Column('position', Integer), - Column('extras', JsonDictType), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('package_resource.id')) - ) - - resource_group = Table( - 'resource_group', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText, ForeignKey('package.id')), - Column('label', UnicodeText), - Column('sort_order', UnicodeText), - Column('extras', JsonDictType), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id')), - ) - - resource_group_revision = Table( - 'resource_group_revision', metadata, - Column('id', UnicodeText, primary_key=True, default=make_uuid), - Column('package_id', UnicodeText, ForeignKey('package.id')), - Column('label', UnicodeText), - Column('sort_order', UnicodeText), - Column('extras', JsonDictType), - Column('state', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', UnicodeText, ForeignKey('resource_group.id')) - ) - - resource_count = migrate_engine.execute('''select count(*) from package_resource''').first()[0] - package_count = migrate_engine.execute('''select count(*) from package''').first()[0] - - - # change field names - resource.c.package_id.alter(name = 'resource_group_id') - resource_revision.c.package_id.alter(name = 'resource_group_id') - - # rename tables - resource.rename('resource') - resource_revision.rename('resource_revision') - - # make new tables - metadata.create_all(migrate_engine) - - # drop all constaints - migrate_engine.execute(''' -ALTER TABLE resource_revision - DROP CONSTRAINT package_resource_revision_pkey; - -ALTER TABLE resource - DROP CONSTRAINT package_resource_revision_id_fkey; - -ALTER TABLE resource_revision - DROP CONSTRAINT package_resource_revision_continuity_id_fkey; - -ALTER TABLE resource_revision - DROP CONSTRAINT package_resource_revision_package_id_fkey; - -ALTER TABLE resource_revision - DROP CONSTRAINT package_resource_revision_revision_id_fkey; - -ALTER TABLE resource - DROP CONSTRAINT package_resource_pkey; - -ALTER TABLE resource - DROP CONSTRAINT package_resource_package_id_fkey; -''') - - - # do data transfer - # give resource group a hashed version of package uuid - # so that we can use the same hash calculation on - # the resource and resource revision table - migrate_engine.execute(''' -insert into resource_group - select - %s, id, 'default', null, null, state, revision_id - from - package; -''' % make_new_uuid("id") -) - - migrate_engine.execute(''' -insert into resource_group_revision - select - id, package_id, 'default', null, null, state, revision_id, id - from - resource_group; -''' -) - - ## update resource table with new ids generated from the - migrate_engine.execute('update resource set resource_group_id = %s' - % make_new_uuid('resource_group_id')) - - migrate_engine.execute('update resource_revision set resource_group_id = %s' - % make_new_uuid('resource_group_id')) - -##add back contraints - - migrate_engine.execute(''' -ALTER TABLE resource - ADD CONSTRAINT resource_pkey PRIMARY KEY (id); - -ALTER TABLE resource_revision - ADD CONSTRAINT resource_revision_pkey PRIMARY KEY (id, revision_id); - -ALTER TABLE resource - ADD CONSTRAINT resource_resource_group_id_fkey FOREIGN KEY (resource_group_id) REFERENCES resource_group(id); - -ALTER TABLE resource - ADD CONSTRAINT resource_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); - -ALTER TABLE resource_revision - ADD CONSTRAINT resource_revision_continuity_id_fkey FOREIGN KEY (continuity_id) REFERENCES resource(id); - -ALTER TABLE resource_revision - ADD CONSTRAINT resource_revision_resource_group_id_fkey FOREIGN KEY (resource_group_id) REFERENCES resource_group(id); - -ALTER TABLE resource_revision - ADD CONSTRAINT resource_revision_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); -''') - - resource_count_after = migrate_engine.execute('''select count(*) from resource''').first()[0] - resource_group_after = migrate_engine.execute('''select count(*) from resource_group''').first()[0] - package_count_after = migrate_engine.execute('''select count(*) from package''').first()[0] - - all_joined = migrate_engine.execute('''select count(*) from package p - join resource_group rg on rg.package_id = p.id - join resource r on r.resource_group_id = rg.id - ''').first()[0] - - all_uuids = migrate_engine.execute(''' - select count(*) from - (select id from resource union - select id from resource_group union - select id from package) sub - ''').first()[0] - - assert resource_count_after == resource_count - assert resource_group_after == package_count - assert package_count_after == package_count - - ## this makes sure all uuids are unique (union dedupes) - assert all_uuids == resource_count + package_count * 2 - - ## this makes sure all uuids are unique (union dedupes) - assert all_joined == resource_count - - - -def make_new_uuid(column_name): - - out = '''substring(md5(%s), 1, 8) || '-' || - substring(md5(%s), 9, 4) || '-' || - substring(md5(%s), 13, 4) || '-' || - substring(md5(%s), 17, 4) || '-' || - substring(md5(%s), 21, 12)''' - - return out % tuple([column_name] * 5) - - - - - -def downgrade(migrate_engine): - raise NotImplementedError() diff --git a/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py b/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py new file mode 100644 index 00000000000..71fddce62d0 --- /dev/null +++ b/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py @@ -0,0 +1,91 @@ +"""035 Harvesting doc versioning + +Revision ID: 81148ccebd6c +Revises: 6c600693af5b +Create Date: 2018-09-04 18:49:01.017635 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '81148ccebd6c' +down_revision = '6c600693af5b' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'harvested_document_revision', + sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('guid', sa.UnicodeText), sa.Column('created', sa.TIMESTAMP), + sa.Column('content', sa.UnicodeText, nullable=False), + sa.Column('source_id', sa.UnicodeText), + sa.Column('package_id', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, nullable=False), + sa.Column('continuity_id', sa.UnicodeText) + ) + + op.add_column('harvested_document', sa.Column('state', sa.UnicodeText)) + op.add_column( + 'harvested_document', sa.Column('revision_id', sa.UnicodeText) + ) + + op.create_primary_key( + 'harvested_document_revision_pkey', 'harvested_document_revision', + ['id', 'revision_id'] + ) + op.create_foreign_key( + 'harvested_document_revision_id_fkey', 'harvested_document', + 'revision', ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_continuity_id_fkey', + 'harvested_document_revision', 'harvested_document', ['continuity_id'], + ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_package_id_fkey', + 'harvested_document_revision', 'package', ['package_id'], ['id'] + ) + + op.create_foreign_key( + 'harvested_document_revision_revision_id_fkey', + 'harvested_document_revision', 'revision', ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_source_id_fkey', + 'harvested_document_revision', 'harvest_source', ['source_id'], ['id'] + ) + + +def downgrade(): + op.drop_constraint( + 'harvested_document_revision_source_id_fkey', + 'harvested_document_revision' + ) + op.drop_constraint( + 'harvested_document_revision_revision_id_fkey', + 'harvested_document_revision' + ) + op.drop_constraint( + 'harvested_document_revision_package_id_fkey', + 'harvested_document_revision' + ) + op.drop_constraint( + 'harvested_document_revision_continuity_id_fkey', + 'harvested_document_revision' + ) + op.drop_constraint( + 'harvested_document_revision_id_fkey', 'harvested_document' + ) + op.drop_constraint( + 'harvested_document_revision_pkey', 'harvested_document_revision' + ) + + op.drop_column('harvested_document', 'state') + op.drop_column('harvested_document', 'revision_id') + + op.drop_table('harvested_document_revision') diff --git a/ckan/migration/versions/035_harvesting_doc_versioning.py b/ckan/migration/versions/035_harvesting_doc_versioning.py deleted file mode 100644 index bbdcae8e9dd..00000000000 --- a/ckan/migration/versions/035_harvesting_doc_versioning.py +++ /dev/null @@ -1,50 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -import uuid -import migrate.changeset -from migrate.changeset.constraint import PrimaryKeyConstraint -from ckan.model.types import JsonDictType - -def upgrade(migrate_engine): - metadata = MetaData(migrate_engine) - - migrate_engine.execute(''' - - CREATE TABLE harvested_document_revision ( - id text NOT NULL, - guid text, - created timestamp without time zone, - content text NOT NULL, - source_id text, - package_id text, - state text, - revision_id text NOT NULL, - continuity_id text - ); - - ALTER TABLE harvested_document - ADD COLUMN state text, - ADD COLUMN revision_id text; - - ALTER TABLE harvested_document_revision - ADD CONSTRAINT harvested_document_revision_pkey PRIMARY KEY (id, revision_id); - - ALTER TABLE harvested_document - ADD CONSTRAINT harvested_document_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); - - ALTER TABLE harvested_document_revision - ADD CONSTRAINT harvested_document_revision_continuity_id_fkey FOREIGN KEY (continuity_id) REFERENCES harvested_document(id); - - ALTER TABLE harvested_document_revision - ADD CONSTRAINT harvested_document_revision_package_id_fkey FOREIGN KEY (package_id) REFERENCES package(id); - - ALTER TABLE harvested_document_revision - ADD CONSTRAINT harvested_document_revision_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); - - ALTER TABLE harvested_document_revision - ADD CONSTRAINT harvested_document_revision_source_id_fkey FOREIGN KEY (source_id) REFERENCES harvest_source(id); - ''' - ) diff --git a/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py b/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py new file mode 100644 index 00000000000..c37f1ba7957 --- /dev/null +++ b/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py @@ -0,0 +1,23 @@ +"""036 Lockdown roles + +Revision ID: ecaa8b38782f +Revises: 81148ccebd6c +Create Date: 2018-09-04 18:49:01.359019 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'ecaa8b38782f' +down_revision = '81148ccebd6c' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/036_lockdown_roles.py b/ckan/migration/versions/036_lockdown_roles.py deleted file mode 100644 index fce82bf659f..00000000000 --- a/ckan/migration/versions/036_lockdown_roles.py +++ /dev/null @@ -1,37 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import datetime -import uuid -from migrate.changeset.constraint import PrimaryKeyConstraint - -def make_uuid(): - return unicode(uuid.uuid4()) - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - role_action = Table('role_action', metadata, autoload=True) - q = role_action.insert(values={'id': make_uuid(), 'role': 'editor', - 'action': 'read-site', 'context': ''}) - migrate_engine.execute(q) - q = role_action.insert(values={'id': make_uuid(), 'role': 'editor', - 'action': 'read-user', 'context': ''}) - migrate_engine.execute(q) - q = role_action.insert(values={'id': make_uuid(), 'role': 'editor', - 'action': 'create-user', 'context': ''}) - migrate_engine.execute(q) - q = role_action.insert(values={'id': make_uuid(), 'role': 'reader', - 'action': 'read-site', 'context': ''}) - migrate_engine.execute(q) - q = role_action.insert(values={'id': make_uuid(), 'role': 'reader', - 'action': 'read-user', 'context': ''}) - migrate_engine.execute(q) - q = role_action.insert(values={'id': make_uuid(), 'role': 'reader', - 'action': 'create-user', 'context': ''}) - migrate_engine.execute(q) - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py b/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py new file mode 100644 index 00000000000..09f9e3cb7e1 --- /dev/null +++ b/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py @@ -0,0 +1,23 @@ +"""037 Role anon_editor + +Revision ID: edcf3b8c3c1b +Revises: ecaa8b38782f +Create Date: 2018-09-04 18:49:01.692660 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'edcf3b8c3c1b' +down_revision = 'ecaa8b38782f' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/037_role_anon_editor.py b/ckan/migration/versions/037_role_anon_editor.py deleted file mode 100644 index e0af0557b00..00000000000 --- a/ckan/migration/versions/037_role_anon_editor.py +++ /dev/null @@ -1,50 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from sqlalchemy.sql import select, and_ -from migrate import * -import logging - -log = logging.getLogger(__name__) - -def upgrade(migrate_engine): - '''#1066 Change Visitor role on System from "reader" to "anon_editor".''' - metadata = MetaData(migrate_engine) - - # get visitor ID - user = Table('user', metadata, autoload=True) - s = select([user.c.id, user.c.name], - user.c.name == u'visitor') - results = migrate_engine.execute(s).fetchall() - if len(results) == 0: - log.debug('No visitor on the system - obviously init hasn\'t been run yet' \ - 'and that will init visitor to an anon_editor') - return - - visitor_id, visitor_name = results[0] - - # find visitor role as reader on system - uor = Table('user_object_role', metadata, autoload=True) - visitor_system_condition = and_(uor.c.context == u'System', - uor.c.user_id == visitor_id) - s = select([uor.c.context, uor.c.user_id, uor.c.role], - visitor_system_condition) - results = migrate_engine.execute(s).fetchall() - if len(results) != 1: - log.warn('Could not find a Right for a Visitor on the System') - return - context, user_id, role = results[0] - if role != 'reader': - log.info('Visitor right for the System is not "reader", so not upgrading it to anon_editor.') - return - - # change visitor role to anon_editor - log.info('Visitor is a "reader" on the System, so upgrading it to "anon_editor".') - sql = uor.update().where(visitor_system_condition).\ - values(role=u'anon_editor') - migrate_engine.execute(sql) - - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/038_delete_migration_tables.py b/ckan/migration/versions/038_delete_migration_tables.py deleted file mode 100644 index 72c90b24100..00000000000 --- a/ckan/migration/versions/038_delete_migration_tables.py +++ /dev/null @@ -1,12 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - - migrate_engine.execute(''' -DROP TABLE harvested_document_revision, harvested_document, harvesting_job, harvest_source; -''') - - - diff --git a/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py b/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py new file mode 100644 index 00000000000..6ef7589caf4 --- /dev/null +++ b/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py @@ -0,0 +1,101 @@ +"""038 Delete migration tables + +Revision ID: fd6622e3d964 +Revises: edcf3b8c3c1b +Create Date: 2018-09-04 18:49:02.023123 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'fd6622e3d964' +down_revision = 'edcf3b8c3c1b' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('harvested_document_revision') + op.drop_table('harvested_document') + op.drop_table('harvesting_job') + op.drop_table('harvest_source') + + +def downgrade(): + op.create_table( + 'harvest_source', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('status', sa.UnicodeText, server_default=u'New'), + sa.Column('url', sa.UnicodeText, unique=True, nullable=False), + sa.Column('description', sa.UnicodeText, default=u''), + sa.Column('user_ref', sa.UnicodeText, default=u''), + sa.Column('publisher_ref', sa.UnicodeText, default=u''), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + ) + + op.create_table( + 'harvesting_job', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('status', sa.UnicodeText, default=u'', nullable=False), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + sa.Column('user_ref', sa.UnicodeText, nullable=False), + sa.Column('report', sa.UnicodeText, default=u''), + sa.Column( + 'source_id', sa.UnicodeText, sa.ForeignKey('harvest_source.id') + ), + ) + + op.create_table( + 'harvested_document', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('created', sa.DateTime), + sa.Column('content', sa.UnicodeText, nullable=False), + sa.Column('source_id', sa.UnicodeText), + sa.Column('package_id', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, nullable=False), + sa.Column('guid', sa.UnicodeText), sa.Column('created', sa.TIMESTAMP), + ) + + op.create_table( + 'harvested_document_revision', + sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('guid', sa.UnicodeText), sa.Column('created', sa.TIMESTAMP), + sa.Column('content', sa.UnicodeText, nullable=False), + sa.Column('source_id', sa.UnicodeText), + sa.Column('package_id', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, nullable=False), + sa.Column('continuity_id', sa.UnicodeText) + ) + op.create_primary_key( + 'harvested_document_revision_pkey', 'harvested_document_revision', + ['id', 'revision_id'] + ) + op.create_foreign_key( + 'harvested_document_revision_id_fkey', 'harvested_document', + 'revision', ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_continuity_id_fkey', + 'harvested_document_revision', 'harvested_document', ['continuity_id'], + ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_package_id_fkey', + 'harvested_document_revision', 'package', ['package_id'], ['id'] + ) + + op.create_foreign_key( + 'harvested_document_revision_revision_id_fkey', + 'harvested_document_revision', 'revision', ['revision_id'], ['id'] + ) + op.create_foreign_key( + 'harvested_document_revision_source_id_fkey', + 'harvested_document_revision', 'harvest_source', ['source_id'], ['id'] + ) diff --git a/ckan/migration/versions/039_add_expired_id_and_dates.py b/ckan/migration/versions/039_add_expired_id_and_dates.py deleted file mode 100644 index c10f5f17c7a..00000000000 --- a/ckan/migration/versions/039_add_expired_id_and_dates.py +++ /dev/null @@ -1,221 +0,0 @@ -# encoding: utf-8 - -from migrate import * -import uuid -import datetime - -def upgrade(migrate_engine): - - id = uuid.uuid4() - - make_missing_revisions = ''' - --- make sure all tables have an entry in the revision_table - -insert into revision values ('%(id)s' , '%(timestamp)s', 'admin', 'Admin: make sure every object has a row in a revision table', 'active'); - -insert into package_tag_revision (id,package_id,tag_id,revision_id,state,continuity_id) select id,package_id,tag_id, '%(id)s' ,state, id from package_tag where package_tag.id not in (select id from package_tag_revision); - -insert into resource_revision (id,resource_group_id,url,format,description,position,revision_id,hash,state,extras,continuity_id) select id,resource_group_id,url,format,description,position, '%(id)s' ,hash,state,extras, id from resource where resource.id not in (select id from resource_revision); - -insert into group_extra_revision (id,group_id,key,value,state,revision_id,continuity_id) select id,group_id,key,value,state, '%(id)s' , id from group_extra where group_extra.id not in (select id from group_extra_revision); - -insert into resource_group_revision (id,package_id,label,sort_order,extras,state,revision_id,continuity_id) select id,package_id,label,sort_order,extras,state, '%(id)s', id from resource_group where resource_group.id not in (select id from resource_group_revision); - -insert into package_extra_revision (id,package_id,key,value,revision_id,state,continuity_id) select id,package_id,key,value, '%(id)s',state, id from package_extra where package_extra.id not in (select id from package_extra_revision); - -insert into package_relationship_revision (id,subject_package_id,object_package_id,type,comment,revision_id,state,continuity_id) select id,subject_package_id,object_package_id,type,comment, '%(id)s',state, id from package_relationship where package_relationship.id not in (select id from package_relationship_revision); - -insert into group_revision (id,name,title,description,created,state,revision_id,continuity_id) select id,name,title,description,created,state, '%(id)s', id from "group" where "group".id not in (select id from group_revision); - -insert into package_revision (id,name,title,url,notes,license_id,revision_id,version,author,author_email,maintainer,maintainer_email,state,continuity_id) select id,name,title,url,notes,license_id, '%(id)s',version,author,author_email,maintainer,maintainer_email,state, id from package where package.id not in (select id from package_revision); - -''' % dict(id=id, timestamp=datetime.datetime.utcnow().isoformat()) - - - update_schema = ''' -ALTER TABLE package_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE package_extra_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE group_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE group_extra_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - - -ALTER TABLE package_group_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE package_tag_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE resource_group_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE resource_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE package_relationship_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - -ALTER TABLE revision - ADD COLUMN approved_timestamp timestamp without time zone; - -create table tmp_expired_id(id text, revision_id text, revision_timestamp timestamp, expired_timestamp timestamp, expired_id text); -create index id_exp on tmp_expired_id(id, revision_id); - ---package revision -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_revision pr join revision r on pr.revision_id = r.id; -update package_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update package_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_package_period on package_revision(revision_timestamp, expired_timestamp, id); -create index idx_package_current on package_revision(current); - ---package extra revision -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_extra_revision pr join revision r on pr.revision_id = r.id; -update package_extra_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update package_extra_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_package_extra_period on package_extra_revision(revision_timestamp, expired_timestamp, id); -create index idx_package_extra_period_package on package_extra_revision(revision_timestamp, expired_timestamp, package_id); -create index idx_package_extra_current on package_extra_revision(current); - ---package group revision -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_group_revision pr join revision r on pr.revision_id = r.id; -update package_group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update package_group_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_package_group_period_package_group on package_group_revision(revision_timestamp, expired_timestamp, package_id, group_id); -create index idx_package_group_current on package_group_revision(current); - - --- package_tags -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_tag_revision pr join revision r on pr.revision_id = r.id; -update package_tag_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update package_tag_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_period_package_tag on package_tag_revision(revision_timestamp, expired_timestamp, package_id, tag_id); -create index idx_package_tag_current on package_tag_revision(current); - --- package relationship -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from package_relationship_revision pr join revision r on pr.revision_id = r.id; -update package_relationship_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update package_relationship_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_period_package_relationship on package_relationship_revision(revision_timestamp, expired_timestamp, object_package_id, subject_package_id); -create index idx_package_relationship_current on package_relationship_revision(current); - --- resource revision -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from resource_revision pr join revision r on pr.revision_id = r.id; -update resource_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update resource_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_resource_period on resource_revision(revision_timestamp, expired_timestamp, id); -create index idx_resource_period_resource_group on resource_revision(revision_timestamp, expired_timestamp, resource_group_id); -create index idx_resource_current on resource_revision(current); - --- resource group revision; -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from resource_group_revision pr join revision r on pr.revision_id = r.id; -update resource_group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update resource_group_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_resource_group_period on resource_group_revision(revision_timestamp, expired_timestamp, id); -create index idx_resource_group_period_package on resource_group_revision(revision_timestamp, expired_timestamp, package_id); -create index idx_resource_group_current on resource_group_revision(current); - ---group revision; -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from group_revision pr join revision r on pr.revision_id = r.id; -update group_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update group_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_group_period on group_revision(revision_timestamp, expired_timestamp, id); -create index idx_group_current on group_revision(current); - ---group extra revision -truncate tmp_expired_id; -insert into tmp_expired_id select pr.id, revision_id, timestamp, lead(timestamp, 1, '9999-12-31') over (partition by pr.id order by timestamp), lead(pr.revision_id) over (partition by pr.id order by timestamp) from group_extra_revision pr join revision r on pr.revision_id = r.id; -update group_extra_revision pr set revision_timestamp = (select revision_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_timestamp = (select expired_timestamp from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id), - expired_id = (select expired_id from tmp_expired_id tmp where tmp.revision_id = pr.revision_id and tmp.id = pr.id); -update group_extra_revision set current = '1' where expired_timestamp = '9999-12-31'; - -create index idx_group_extra_period on group_extra_revision(revision_timestamp, expired_timestamp, id); -create index idx_group_extra_period_group on group_extra_revision(revision_timestamp, expired_timestamp, group_id); -create index idx_group_extra_current on group_extra_revision(current); - -drop table tmp_expired_id; - --- change state of revision tables - -update revision set approved_timestamp = timestamp; -''' - - migrate_engine.execute('begin; ' + make_missing_revisions + update_schema + ' commit;') - - for table in ['package', 'resource', 'resource_group', 'package_extra', - 'package_tag', 'package_relationship', 'group', 'group_extra']: - count = migrate_engine.execute('''select count(*) from "%s"''' % table).first()[0] - revision_expired_id_count = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.expired_id is null''' % (table, table)).first()[0] - revision_expired_data_count = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.expired_timestamp = '9999-12-31' ''' % (table, table)).first()[0] - revision_current = migrate_engine.execute('''select count(*) from %s_revision where %s_revision.current = '1' ''' % (table, table)).first()[0] - assert count == revision_expired_id_count - assert count == revision_expired_data_count - assert count == revision_current - - diff --git a/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py b/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py new file mode 100644 index 00000000000..0ddaf80778c --- /dev/null +++ b/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py @@ -0,0 +1,156 @@ +"""039 Add expired id and_dates + +Revision ID: cca459c76d45 +Revises: fd6622e3d964 +Create Date: 2018-09-04 18:49:02.364964 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'cca459c76d45' +down_revision = 'fd6622e3d964' +branch_labels = None +depends_on = None + +tables = [ + 'package_revision', 'package_extra_revision', 'group_revision', + 'group_extra_revision', 'package_group_revision', 'package_tag_revision', + 'resource_group_revision', 'resource_revision', + 'package_relationship_revision' +] +indexes = ( + ( + 'package_revision', ( + ( + 'idx_package_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ('idx_package_current', ('current', )), + ) + ), + ( + 'package_extra_revision', ( + ( + 'idx_package_extra_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ( + 'idx_package_extra_period_package', + ('revision_timestamp', 'expired_timestamp', 'package_id') + ), + ('idx_package_extra_current', ('current', )), + ) + ), + ( + 'package_group_revision', ( + ( + 'idx_package_group_period_package_group', ( + 'revision_timestamp', 'expired_timestamp', 'package_id', + 'group_id' + ) + ), + ('idx_package_group_current', ('current', )), + ) + ), + ( + 'package_tag_revision', ( + ( + 'idx_period_package_tag', ( + 'revision_timestamp', 'expired_timestamp', 'package_id', + 'tag_id' + ) + ), + ('idx_package_tag_current', ('current', )), + ) + ), + ( + 'package_relationship_revision', ( + ( + 'idx_period_package_relationship', ( + 'revision_timestamp', 'expired_timestamp', + 'object_package_id', 'subject_package_id' + ) + ), + ('idx_package_relationship_current', ('current', )), + ) + ), + ( + 'resource_revision', ( + ( + 'idx_resource_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ( + 'idx_resource_period_resource_group', ( + 'revision_timestamp', 'expired_timestamp', + 'resource_group_id' + ) + ), + ('idx_resource_current', ('current', )), + ) + ), + ( + 'resource_group_revision', ( + ( + 'idx_resource_group_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ( + 'idx_resource_group_period_package', + ('revision_timestamp', 'expired_timestamp', 'package_id') + ), + ('idx_resource_group_current', ('current', )), + ) + ), + ( + 'group_revision', ( + ( + 'idx_group_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ('idx_group_current', ('current', )), + ) + ), + ( + 'group_extra_revision', ( + ( + 'idx_group_extra_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ( + 'idx_group_extra_period_group', + ('revision_timestamp', 'expired_timestamp', 'group_id') + ), + ('idx_group_extra_current', ('current', )), + ) + ), +) + + +def upgrade(): + + for table in tables: + op.add_column(table, sa.Column('expired_id', sa.UnicodeText)) + op.add_column(table, sa.Column('revision_timestamp', sa.TIMESTAMP)) + op.add_column(table, sa.Column('expired_timestamp', sa.TIMESTAMP)) + op.add_column(table, sa.Column('current', sa.Boolean)) + op.add_column('revision', sa.Column('approved_timestamp', sa.TIMESTAMP)) + + for table, items in indexes: + for index, columns in items: + op.create_index(index, table, list(columns)) + + +def downgrade(): + for table, items in reversed(indexes): + for index, _ in items: + op.drop_index(index, table) + + op.drop_column('revision', 'approved_timestamp') + for table in reversed(tables): + op.drop_column(table, 'expired_id') + op.drop_column(table, 'revision_timestamp') + op.drop_column(table, 'expired_timestamp') + op.drop_column(table, 'current') diff --git a/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py b/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py new file mode 100644 index 00000000000..7cc2c0ef66e --- /dev/null +++ b/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py @@ -0,0 +1,23 @@ +"""040 Reset key on user + +Revision ID: 500a08f4818e +Revises: cca459c76d45 +Create Date: 2018-09-04 18:49:02.701370 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '500a08f4818e' +down_revision = 'cca459c76d45' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('user', sa.Column('reset_key', sa.UnicodeText)) + + +def downgrade(): + op.drop_column('user', 'reset_key') diff --git a/ckan/migration/versions/040_reset_key_on_user.py b/ckan/migration/versions/040_reset_key_on_user.py deleted file mode 100644 index 4acdaa2f3b4..00000000000 --- a/ckan/migration/versions/040_reset_key_on_user.py +++ /dev/null @@ -1,15 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - user_table = Table('user', metadata, autoload=True) - reset_key_col = Column('reset_key', UnicodeText) - reset_key_col.create(user_table) - -def downgrade(migrate_engine): - raise NotImplementedError() - diff --git a/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py b/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py new file mode 100644 index 00000000000..1fcdd72f884 --- /dev/null +++ b/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py @@ -0,0 +1,43 @@ +"""041 Resource new fields + +Revision ID: 6817d4e3bdc3 +Revises: 500a08f4818e +Create Date: 2018-09-04 18:49:03.042528 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6817d4e3bdc3' +down_revision = '500a08f4818e' +branch_labels = None +depends_on = None + + +def upgrade(): + for table in ('resource', 'resource_revision'): + op.add_column(table, sa.Column('name', sa.UnicodeText)) + op.add_column(table, sa.Column('resource_type', sa.UnicodeText)) + op.add_column(table, sa.Column('mimetype', sa.UnicodeText)) + op.add_column(table, sa.Column('mimetype_inner', sa.UnicodeText)) + op.add_column(table, sa.Column('size', sa.BigInteger)) + op.add_column(table, sa.Column('last_modified', sa.TIMESTAMP)) + op.add_column(table, sa.Column('cache_url', sa.UnicodeText)) + op.add_column(table, sa.Column('cache_last_updated', sa.TIMESTAMP)) + op.add_column(table, sa.Column('webstore_url', sa.UnicodeText)) + op.add_column(table, sa.Column('webstore_last_updated', sa.TIMESTAMP)) + + +def downgrade(): + for table in ('resource', 'resource_revision'): + op.drop_column(table, 'name') + op.drop_column(table, 'resource_type') + op.drop_column(table, 'mimetype') + op.drop_column(table, 'mimetype_inner') + op.drop_column(table, 'size') + op.drop_column(table, 'last_modified') + op.drop_column(table, 'cache_url') + op.drop_column(table, 'cache_last_updated') + op.drop_column(table, 'webstore_url') + op.drop_column(table, 'webstore_last_updated') diff --git a/ckan/migration/versions/041_resource_new_fields.py b/ckan/migration/versions/041_resource_new_fields.py deleted file mode 100644 index 1acd6a51dce..00000000000 --- a/ckan/migration/versions/041_resource_new_fields.py +++ /dev/null @@ -1,34 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - begin; - ALTER TABLE resource - ADD COLUMN name text, - ADD COLUMN resource_type text, - ADD COLUMN mimetype text, - ADD COLUMN mimetype_inner text, - ADD COLUMN "size" bigint, - ADD COLUMN last_modified timestamp without time zone, - ADD COLUMN cache_url text, - ADD COLUMN cache_last_updated timestamp without time zone, - ADD COLUMN webstore_url text, - ADD COLUMN webstore_last_updated timestamp without time zone; - - ALTER TABLE resource_revision - ADD COLUMN name text, - ADD COLUMN resource_type text, - ADD COLUMN mimetype text, - ADD COLUMN mimetype_inner text, - ADD COLUMN "size" bigint, - ADD COLUMN last_modified timestamp without time zone, - ADD COLUMN cache_url text, - ADD COLUMN cache_last_updated timestamp without time zone, - ADD COLUMN webstore_url text, - ADD COLUMN webstore_last_updated timestamp without time zone; - commit; - ''' - ) diff --git a/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py b/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py new file mode 100644 index 00000000000..26e10d612b1 --- /dev/null +++ b/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py @@ -0,0 +1,33 @@ +"""042 User revision indexes + +Revision ID: da65e2877034 +Revises: 6817d4e3bdc3 +Create Date: 2018-09-04 18:49:03.380103 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'da65e2877034' +down_revision = '6817d4e3bdc3' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index('idx_revision_author', 'revision', ['author']) + op.create_index('idx_openid', 'user', ['openid']) + op.create_index( + 'idx_user_name_index', 'user', [ + sa.text( + '''(CASE WHEN ("user".fullname IS NULL OR "user".fullname = '') THEN "user".name ELSE "user".fullname END)''' + ) + ] + ) + + +def downgrade(): + op.drop_index('idx_user_name_index', 'user') + op.drop_index('idx_openid', 'user') + op.drop_index('idx_revision_author', 'revision') diff --git a/ckan/migration/versions/042_user_revision_indexes.py b/ckan/migration/versions/042_user_revision_indexes.py deleted file mode 100644 index 0be68b3885a..00000000000 --- a/ckan/migration/versions/042_user_revision_indexes.py +++ /dev/null @@ -1,15 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - BEGIN; - CREATE INDEX idx_revision_author ON "revision" (author); - CREATE INDEX idx_openid ON "user" (openid); - CREATE INDEX "idx_user_name_index" on "user"((CASE WHEN ("user".fullname IS NULL OR "user".fullname = '') THEN "user".name ELSE "user".fullname END)); - COMMIT; - ''' - ) - - diff --git a/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py b/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py new file mode 100644 index 00000000000..fb7411a8cda --- /dev/null +++ b/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py @@ -0,0 +1,31 @@ +"""043 Drop postgres search + +Revision ID: bd38cd6502b2 +Revises: da65e2877034 +Create Date: 2018-09-04 18:49:03.717678 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'bd38cd6502b2' +down_revision = 'da65e2877034' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('package_search') + + +def downgrade(): + op.create_table( + 'package_search', + sa.Column( + 'package_id', + sa.UnicodeText, + sa.ForeignKey('package.id'), + primary_key=True + ), sa.Column('search_vector', sa.dialects.postgresql.TSVECTOR) + ) diff --git a/ckan/migration/versions/043_drop_postgres_search.py b/ckan/migration/versions/043_drop_postgres_search.py deleted file mode 100644 index b44089a9f36..00000000000 --- a/ckan/migration/versions/043_drop_postgres_search.py +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - package_search_table = Table('package_search', metadata, autoload=True) - package_search_table.drop() -## migrate_engine.execute(''' -##DROP TABLE package_search; -##''') diff --git a/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py b/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py new file mode 100644 index 00000000000..eb3367b4ef7 --- /dev/null +++ b/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py @@ -0,0 +1,41 @@ +"""044 Add task status + +Revision ID: 4190eeeb8d73 +Revises: bd38cd6502b2 +Create Date: 2018-09-04 18:49:04.084036 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '4190eeeb8d73' +down_revision = 'bd38cd6502b2' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'task_status', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('entity_id', sa.UnicodeText, nullable=False), + sa.Column('entity_type', sa.UnicodeText, nullable=False), + sa.Column('task_type', sa.UnicodeText, nullable=False), + sa.Column('key', sa.UnicodeText, nullable=False), + sa.Column('value', sa.UnicodeText, nullable=False), + sa.Column('state', sa.UnicodeText), sa.Column('error', sa.UnicodeText), + sa.Column('last_updated', sa.TIMESTAMP) + ) + op.create_primary_key('task_status_pkey', 'task_status', ['id']) + op.create_unique_constraint( + 'task_status_entity_id_task_type_key_key', 'task_status', + ['entity_id', 'task_type', 'key'] + ) + + +def downgrade(): + op.drop_constraint('task_status_pkey', 'task_status') + op.drop_constraint( + 'task_status_entity_id_task_type_key_key', 'task_status' + ) + op.drop_table('task_status') diff --git a/ckan/migration/versions/044_add_task_status.py b/ckan/migration/versions/044_add_task_status.py deleted file mode 100644 index c9a950e1a6d..00000000000 --- a/ckan/migration/versions/044_add_task_status.py +++ /dev/null @@ -1,26 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - CREATE TABLE task_status ( - id text NOT NULL, - entity_id text NOT NULL, - entity_type text NOT NULL, - task_type text NOT NULL, - "key" text NOT NULL, - "value" text NOT NULL, - "state" text, - "error" text, - last_updated timestamp without time zone - ); - - ALTER TABLE task_status - ADD CONSTRAINT task_status_pkey PRIMARY KEY (id); - - ALTER TABLE task_status - ADD CONSTRAINT task_status_entity_id_task_type_key_key UNIQUE (entity_id, task_type, key); - ''' - ) diff --git a/ckan/migration/versions/045_54e3f155d945_user_name_unique.py b/ckan/migration/versions/045_54e3f155d945_user_name_unique.py new file mode 100644 index 00000000000..42a7b006538 --- /dev/null +++ b/ckan/migration/versions/045_54e3f155d945_user_name_unique.py @@ -0,0 +1,23 @@ +"""045 User name unique + +Revision ID: 54e3f155d945 +Revises: 4190eeeb8d73 +Create Date: 2018-09-04 18:49:04.437304 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '54e3f155d945' +down_revision = '4190eeeb8d73' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_unique_constraint('user_name_key', 'user', ['name']) + + +def downgrade(): + op.drop_constraint('user_name_key', 'user') diff --git a/ckan/migration/versions/045_user_name_unique.py b/ckan/migration/versions/045_user_name_unique.py deleted file mode 100644 index c8873182b35..00000000000 --- a/ckan/migration/versions/045_user_name_unique.py +++ /dev/null @@ -1,19 +0,0 @@ -# encoding: utf-8 - -import warnings - -from sqlalchemy import exc as sa_exc -from sqlalchemy import * -from migrate import * -from migrate.changeset.constraint import UniqueConstraint - -def upgrade(migrate_engine): - # ignore reflection warnings - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=sa_exc.SAWarning) - metadata = MetaData() - metadata.bind = migrate_engine - user_table = Table('user', metadata, autoload=True) - # name_column = user_table.c.name - unique_name_constraint = UniqueConstraint('name', table=user_table) - unique_name_constraint.create() diff --git a/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py b/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py new file mode 100644 index 00000000000..062d2a2d9af --- /dev/null +++ b/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py @@ -0,0 +1,68 @@ +"""046 Drop changesets + +Revision ID: b69e9b80396f +Revises: 54e3f155d945 +Create Date: 2018-09-04 18:49:04.791120 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'b69e9b80396f' +down_revision = '54e3f155d945' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('change') + op.drop_table('changemask') + op.drop_table('changeset') + + +def downgrade(): + op.create_table( + 'changeset', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('closes_id', sa.UnicodeText, nullable=True), + sa.Column('follows_id', sa.UnicodeText, nullable=True), + sa.Column('meta', sa.UnicodeText, nullable=True), + sa.Column('branch', sa.UnicodeText, nullable=True), + sa.Column( + 'timestamp', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + sa.Column('is_working', sa.Boolean, default=False), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + nullable=True + ), + sa.Column( + 'added_here', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + ) + + op.create_table( + 'change', + sa.Column('ref', sa.UnicodeText, nullable=True), + sa.Column('diff', sa.UnicodeText, nullable=True), + sa.Column( + 'changeset_id', sa.UnicodeText, sa.ForeignKey('changeset.id') + ), + ) + + op.create_table( + 'changemask', + sa.Column('ref', sa.UnicodeText, primary_key=True), + sa.Column( + 'timestamp', + sa.DateTime, + server_default=sa.func.current_timestamp() + ), + ) diff --git a/ckan/migration/versions/046_drop_changesets.py b/ckan/migration/versions/046_drop_changesets.py deleted file mode 100644 index 2715b32265c..00000000000 --- a/ckan/migration/versions/046_drop_changesets.py +++ /dev/null @@ -1,12 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * -import migrate.changeset - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - for table_name in ['change', 'changemask', 'changeset']: - table = Table(table_name, metadata, autoload=True) - table.drop() diff --git a/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py b/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py new file mode 100644 index 00000000000..7467535727a --- /dev/null +++ b/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py @@ -0,0 +1,168 @@ +"""047 Rename package_group_member + +Revision ID: 883a7c406926 +Revises: b69e9b80396f +Create Date: 2018-09-04 18:49:05.130215 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '883a7c406926' +down_revision = 'b69e9b80396f' +branch_labels = None +depends_on = None + + +def upgrade(): + op.rename_table('package_group', 'member') + op.rename_table('package_group_revision', 'member_revision') + + op.alter_column( + 'member', 'package_id', new_column_name='table_id', nullable=False + ) + op.alter_column( + 'member_revision', + 'package_id', + new_column_name='table_id', + nullable=False + ) + + op.drop_constraint('package_group_revision_pkey', 'member_revision') + op.drop_constraint( + 'package_group_revision_continuity_id_fkey', 'member_revision' + ) + op.drop_constraint( + 'package_group_revision_group_id_fkey', 'member_revision' + ) + op.drop_constraint( + 'package_group_revision_package_id_fkey', 'member_revision' + ) + op.drop_constraint( + 'package_group_revision_revision_id_fkey', 'member_revision' + ) + + op.drop_constraint('package_group_pkey', 'member') + op.drop_constraint('package_group_group_id_fkey', 'member') + op.drop_constraint('package_group_package_id_fkey', 'member') + op.drop_constraint('package_group_revision_id_fkey', 'member') + + op.add_column( + 'member', sa.Column('table_name', sa.UnicodeText, nullable=False) + ) + op.add_column( + 'member', sa.Column('capacity', sa.UnicodeText, nullable=False) + ) + op.add_column( + 'member_revision', + sa.Column('table_name', sa.UnicodeText, nullable=False) + ) + op.add_column( + 'member_revision', + sa.Column('capacity', sa.UnicodeText, nullable=False) + ) + + op.create_primary_key('member_pkey', 'member', ['id']) + op.create_primary_key( + 'member_revision_pkey', 'member_revision', ['id', 'revision_id'] + ) + + op.create_foreign_key( + 'member_group_id_fkey', 'member', 'group', ['group_id'], ['id'] + ) + op.create_foreign_key( + 'member_revision_id_fkey', 'member', 'revision', ['revision_id'], + ['id'] + ) + op.create_foreign_key( + 'member_revision_continuity_id_fkey', 'member_revision', 'member', + ['continuity_id'], ['id'] + ) + op.create_foreign_key( + 'member_revision_group_id_fkey', 'member_revision', 'group', + ['group_id'], ['id'] + ) + op.create_foreign_key( + 'member_revision_revision_id_fkey', 'member_revision', 'revision', + ['revision_id'], ['id'] + ) + + op.add_column('group', sa.Column('type', sa.UnicodeText, nullable=False)) + op.add_column( + 'group_revision', sa.Column('type', sa.UnicodeText, nullable=False) + ) + + op.add_column('package', sa.Column('type', sa.UnicodeText)) + op.add_column('package_revision', sa.Column('type', sa.UnicodeText)) + + +def downgrade(): + op.drop_column('package', 'type') + op.drop_column('package_revision', 'type') + + op.drop_column('group', 'type') + op.drop_column('group_revision', 'type') + + op.drop_constraint('member_revision_continuity_id_fkey', 'member_revision') + op.drop_constraint('member_revision_group_id_fkey', 'member_revision') + op.drop_constraint('member_revision_revision_id_fkey', 'member_revision') + op.drop_constraint('member_group_id_fkey', 'member') + op.drop_constraint('member_revision_id_fkey', 'member') + + op.drop_constraint('member_revision_pkey', 'member_revision') + op.drop_constraint('member_pkey', 'member') + + op.drop_column('member_revision', 'table_name') + op.drop_column('member_revision', 'capacity') + + op.drop_column('member', 'table_name') + op.drop_column('member', 'capacity') + + op.alter_column( + 'member_revision', + 'table_id', + new_column_name='package_id', + nullable=True + ) + op.alter_column( + 'member', 'table_id', new_column_name='package_id', nullable=True + ) + + op.create_primary_key('package_group_pkey', 'member', ['id']) + op.create_primary_key( + 'package_group_revision_pkey', 'member_revision', + ['id', 'revision_id'] + ) + + op.create_foreign_key( + 'package_group_group_id_fkey', 'member', 'group', ['group_id'], ['id'] + ) + op.create_foreign_key( + 'package_group_package_id_fkey', 'member', 'package', ['package_id'], + ['id'] + ) + op.create_foreign_key( + 'package_group_revision_id_fkey', 'member', 'revision', + ['revision_id'], ['id'] + ) + + op.create_foreign_key( + 'package_group_revision_continuity_id_fkey', 'member_revision', + 'member', ['continuity_id'], ['id'] + ) + op.create_foreign_key( + 'package_group_revision_group_id_fkey', 'member_revision', 'group', + ['group_id'], ['id'] + ) + op.create_foreign_key( + 'package_group_revision_package_id_fkey', 'member_revision', 'package', + ['package_id'], ['id'] + ) + op.create_foreign_key( + 'package_group_revision_revision_id_fkey', 'member_revision', + 'revision', ['revision_id'], ['id'] + ) + + op.rename_table('member_revision', 'package_group_revision') + op.rename_table('member', 'package_group') diff --git a/ckan/migration/versions/047_rename_package_group_member.py b/ckan/migration/versions/047_rename_package_group_member.py deleted file mode 100644 index 1d377739e9c..00000000000 --- a/ckan/migration/versions/047_rename_package_group_member.py +++ /dev/null @@ -1,110 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' -BEGIN; -alter table package_group RENAME to member; -alter table package_group_revision RENAME to member_revision; -alter table member RENAME column package_id to table_id; -alter table member_revision RENAME column package_id to table_id; - -alter table member ALTER column table_id set NOT NULL; -alter table member_revision ALTER column table_id set NOT NULL; - -ALTER TABLE member_revision - DROP CONSTRAINT package_group_revision_pkey; - -ALTER TABLE member_revision - DROP CONSTRAINT package_group_revision_continuity_id_fkey; - -ALTER TABLE member_revision - DROP CONSTRAINT package_group_revision_group_id_fkey; - -ALTER TABLE member_revision - DROP CONSTRAINT package_group_revision_package_id_fkey; - -ALTER TABLE member_revision - DROP CONSTRAINT package_group_revision_revision_id_fkey; - -ALTER TABLE "member" - DROP CONSTRAINT package_group_pkey; - -ALTER TABLE "member" - DROP CONSTRAINT package_group_group_id_fkey; - -ALTER TABLE "member" - DROP CONSTRAINT package_group_package_id_fkey; - -ALTER TABLE "member" - DROP CONSTRAINT package_group_revision_id_fkey; - - -ALTER TABLE "member" - ADD COLUMN table_name text; -ALTER TABLE "member" - ADD COLUMN capacity text; - -ALTER TABLE "member_revision" - ADD COLUMN table_name text; -ALTER TABLE "member_revision" - ADD COLUMN capacity text; - -ALTER TABLE "member" - ADD CONSTRAINT member_pkey PRIMARY KEY (id); - -ALTER TABLE member_revision - ADD CONSTRAINT member_revision_pkey PRIMARY KEY (id, revision_id); - -ALTER TABLE "member" - ADD CONSTRAINT member_group_id_fkey FOREIGN KEY (group_id) REFERENCES "group"(id); - -ALTER TABLE "member" - ADD CONSTRAINT member_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); - -ALTER TABLE member_revision - ADD CONSTRAINT member_revision_continuity_id_fkey FOREIGN KEY (continuity_id) REFERENCES member(id); - -ALTER TABLE member_revision - ADD CONSTRAINT member_revision_group_id_fkey FOREIGN KEY (group_id) REFERENCES "group"(id); - -ALTER TABLE member_revision - ADD CONSTRAINT member_revision_revision_id_fkey FOREIGN KEY (revision_id) REFERENCES revision(id); - -ALTER TABLE "group" - ADD COLUMN "type" text; -ALTER TABLE "group_revision" - ADD COLUMN "type" text; - -update member set table_name = 'package', capacity = 'public'; -update member_revision set table_name = 'package', capacity = 'public'; - -update "group" set type = 'group'; -update group_revision set type = 'group'; - - -ALTER TABLE "member" - ALTER COLUMN table_name set not null; -ALTER TABLE "member" - ALTER COLUMN capacity set not null; - -ALTER TABLE member_revision - ALTER COLUMN table_name set not null; -ALTER TABLE "member_revision" - ALTER COLUMN capacity set not null; - -ALTER TABLE "group" - ALTER COLUMN "type" set not null; -ALTER TABLE "group_revision" - ALTER COLUMN "type" set not null; - -ALTER TABLE "package" - ADD COLUMN "type" text; -ALTER TABLE "package_revision" - ADD COLUMN "type" text; - - -COMMIT; - ''' - ) diff --git a/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py b/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py new file mode 100644 index 00000000000..95c201da4be --- /dev/null +++ b/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py @@ -0,0 +1,50 @@ +"""048 Add activity streams tables + +Revision ID: 4a7011172b3f +Revises: 883a7c406926 +Create Date: 2018-09-04 18:49:05.463765 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '4a7011172b3f' +down_revision = '883a7c406926' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'activity', + sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('timestamp', sa.TIMESTAMP), + sa.Column('user_id', sa.UnicodeText), + sa.Column('object_id', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText), + sa.Column('activity_type', sa.UnicodeText), + sa.Column('data', sa.UnicodeText), + ) + + op.create_table( + 'activity_detail', + sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('activity_id', sa.UnicodeText, nullable=False), + sa.Column('object_id', sa.UnicodeText), + sa.Column('object_type', sa.UnicodeText), + sa.Column('activity_type', sa.UnicodeText), + sa.Column('data', sa.UnicodeText), + ) + + op.create_primary_key('activity_pkey', 'activity', ['id']) + op.create_primary_key('activity_detail_pkey', 'activity_detail', ['id']) + op.create_foreign_key( + 'activity_detail_activity_id_fkey', 'activity_detail', 'activity', + ['activity_id'], ['id'] + ) + + +def downgrade(): + op.drop_table('activity_detail') + op.drop_table('activity') diff --git a/ckan/migration/versions/048_add_activity_streams_tables.py b/ckan/migration/versions/048_add_activity_streams_tables.py deleted file mode 100644 index 079cc6fc3fa..00000000000 --- a/ckan/migration/versions/048_add_activity_streams_tables.py +++ /dev/null @@ -1,37 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -CREATE TABLE activity ( - id text NOT NULL, - timestamp timestamp without time zone, - user_id text, - object_id text, - revision_id text, - activity_type text, - data text -); - -CREATE TABLE activity_detail ( - id text NOT NULL, - activity_id text NOT NULL, - object_id text, - object_type text, - activity_type text, - data text -); - -ALTER TABLE activity - ADD CONSTRAINT activity_pkey PRIMARY KEY (id); - -ALTER TABLE activity_detail - ADD CONSTRAINT activity_detail_pkey PRIMARY KEY (id); - -ALTER TABLE activity_detail - ADD CONSTRAINT activity_detail_activity_id_fkey FOREIGN KEY (activity_id) REFERENCES activity(id); - ''') diff --git a/ckan/migration/versions/049_add_group_approval_status.py b/ckan/migration/versions/049_add_group_approval_status.py deleted file mode 100644 index a327bf2bf03..00000000000 --- a/ckan/migration/versions/049_add_group_approval_status.py +++ /dev/null @@ -1,19 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' -BEGIN; -ALTER TABLE "group" - ADD COLUMN approval_status text; - -ALTER TABLE group_revision - ADD COLUMN approval_status text; - -update "group" set approval_status = 'approved'; -update group_revision set approval_status = 'approved'; - -COMMIT; - ''' - ) diff --git a/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py b/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py new file mode 100644 index 00000000000..10717202768 --- /dev/null +++ b/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py @@ -0,0 +1,27 @@ +"""049 Add group approval status + +Revision ID: e0c06c2177b5 +Revises: 4a7011172b3f +Create Date: 2018-09-04 18:49:05.797861 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e0c06c2177b5' +down_revision = '4a7011172b3f' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('group', sa.Column('approval_status', sa.UnicodeText)) + op.add_column( + 'group_revision', sa.Column('approval_status', sa.UnicodeText) + ) + + +def downgrade(): + op.drop_column('group', 'approval_status') + op.drop_column('group_revision', 'approval_status') diff --git a/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py b/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py new file mode 100644 index 00000000000..9894644156c --- /dev/null +++ b/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py @@ -0,0 +1,29 @@ +"""050 Term translation table + +Revision ID: 01a6b058cb7f +Revises: e0c06c2177b5 +Create Date: 2018-09-04 18:49:06.143050 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '01a6b058cb7f' +down_revision = 'e0c06c2177b5' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'term_translation', sa.Column('term', sa.UnicodeText, nullable=False), + sa.Column('term_translation', sa.UnicodeText, nullable=False), + sa.Column('lang_code', sa.UnicodeText, nullable=False) + ) + op.create_index('term_lang', 'term_translation', ['term', 'lang_code']) + op.create_index('term', 'term_translation', ['term']) + + +def downgrade(): + op.drop_table('term_translation') diff --git a/ckan/migration/versions/050_term_translation_table.py b/ckan/migration/versions/050_term_translation_table.py deleted file mode 100644 index f87608c08c2..00000000000 --- a/ckan/migration/versions/050_term_translation_table.py +++ /dev/null @@ -1,22 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - CREATE TABLE term_translation ( - term text NOT NULL, - term_translation text NOT NULL, - lang_code text NOT NULL - ); - - create index term_lang on term_translation(term, lang_code); - create index term on term_translation(term); - ''' - ) - - - - - diff --git a/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py b/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py new file mode 100644 index 00000000000..4557f84a4a2 --- /dev/null +++ b/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py @@ -0,0 +1,43 @@ +"""051 Add tag vocabulary + +Revision ID: a4fb0d85ced6 +Revises: 01a6b058cb7f +Create Date: 2018-09-04 18:49:06.480087 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'a4fb0d85ced6' +down_revision = '01a6b058cb7f' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint('tag_name_key', 'tag') + op.create_table( + 'vocabulary', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('name', sa.String(100), nullable=False) + ) + + op.add_column('tag', sa.Column('vocabulary_id', sa.String(100))) + op.create_primary_key('vocabulary_pkey', 'vocabulary', ['id']) + op.create_unique_constraint( + 'tag_name_vocabulary_id_key', 'tag', ['name', 'vocabulary_id'] + ) + op.create_foreign_key( + 'tag_vocabulary_id_fkey', 'tag', 'vocabulary', ['vocabulary_id'], + ['id'] + ) + op.create_unique_constraint('vocabulary_name_key', 'vocabulary', ['name']) + + +def downgrade(): + op.drop_constraint('tag_name_vocabulary_id_key', 'tag') + op.drop_constraint('tag_vocabulary_id_fkey', 'tag') + + op.drop_column('tag', 'vocabulary_id') + op.drop_table('vocabulary') + op.create_unique_constraint('tag_name_key', 'tag', ['name']) diff --git a/ckan/migration/versions/051_add_tag_vocabulary.py b/ckan/migration/versions/051_add_tag_vocabulary.py deleted file mode 100644 index 746eff4fd3c..00000000000 --- a/ckan/migration/versions/051_add_tag_vocabulary.py +++ /dev/null @@ -1,31 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE tag - DROP CONSTRAINT tag_name_key; - - CREATE TABLE vocabulary ( - id text NOT NULL, - name character varying(100) NOT NULL - ); - - ALTER TABLE tag - ADD COLUMN vocabulary_id character varying(100); - - ALTER TABLE vocabulary - ADD CONSTRAINT vocabulary_pkey PRIMARY KEY (id); - - ALTER TABLE tag - ADD CONSTRAINT tag_name_vocabulary_id_key UNIQUE (name, vocabulary_id); - - ALTER TABLE tag - ADD CONSTRAINT tag_vocabulary_id_fkey FOREIGN KEY (vocabulary_id) REFERENCES vocabulary(id); - - ALTER TABLE vocabulary - ADD CONSTRAINT vocabulary_name_key UNIQUE (name); - ''' - ) diff --git a/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py b/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py new file mode 100644 index 00000000000..0a66b12607b --- /dev/null +++ b/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py @@ -0,0 +1,23 @@ +"""052 Update member capacities + +Revision ID: ba693d64c6d7 +Revises: a4fb0d85ced6 +Create Date: 2018-09-04 18:49:06.885179 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'ba693d64c6d7' +down_revision = 'a4fb0d85ced6' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/052_update_member_capacities.py b/ckan/migration/versions/052_update_member_capacities.py deleted file mode 100644 index 971b50ecb2d..00000000000 --- a/ckan/migration/versions/052_update_member_capacities.py +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(""" - BEGIN; - UPDATE member SET capacity='public' WHERE capacity='member' - AND table_name='package'; - UPDATE member_revision SET capacity='public' WHERE capacity='member' - AND table_name='package'; - COMMIT; - """ - ) diff --git a/ckan/migration/versions/053_9d051a099097_add_group_logo.py b/ckan/migration/versions/053_9d051a099097_add_group_logo.py new file mode 100644 index 00000000000..e0219cfc481 --- /dev/null +++ b/ckan/migration/versions/053_9d051a099097_add_group_logo.py @@ -0,0 +1,25 @@ +"""053 Add group logo + +Revision ID: 9d051a099097 +Revises: ba693d64c6d7 +Create Date: 2018-09-04 18:49:07.217429 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '9d051a099097' +down_revision = 'ba693d64c6d7' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('group', sa.Column('image_url', sa.UnicodeText)) + op.add_column('group_revision', sa.Column('image_url', sa.UnicodeText)) + + +def downgrade(): + op.drop_column('group', 'image_url') + op.drop_column('group_revision', 'image_url') diff --git a/ckan/migration/versions/053_add_group_logo.py b/ckan/migration/versions/053_add_group_logo.py deleted file mode 100644 index fc1fe3bdbf9..00000000000 --- a/ckan/migration/versions/053_add_group_logo.py +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE "group" - ADD COLUMN image_url text; - - ALTER TABLE group_revision - ADD COLUMN image_url text; - ''' - ) diff --git a/ckan/migration/versions/054_add_resource_created_date.py b/ckan/migration/versions/054_add_resource_created_date.py deleted file mode 100644 index 2f6a0cd993d..00000000000 --- a/ckan/migration/versions/054_add_resource_created_date.py +++ /dev/null @@ -1,11 +0,0 @@ -# encoding: utf-8 - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE resource - ADD COLUMN created timestamp without time zone; - - ALTER TABLE resource_revision - ADD COLUMN created timestamp without time zone; - ''' - ) diff --git a/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py b/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py new file mode 100644 index 00000000000..0c33c3608eb --- /dev/null +++ b/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py @@ -0,0 +1,25 @@ +"""054 Add resource created date + +Revision ID: da21b38da4db +Revises: 9d051a099097 +Create Date: 2018-09-04 18:49:07.555419 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'da21b38da4db' +down_revision = '9d051a099097' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('resource', sa.Column('created', sa.TIMESTAMP)) + op.add_column('resource_revision', sa.Column('created', sa.TIMESTAMP)) + + +def downgrade(): + op.drop_column('resource', 'created') + op.drop_column('resource_revision', 'created') diff --git a/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py b/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py new file mode 100644 index 00000000000..e8d83abeafd --- /dev/null +++ b/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py @@ -0,0 +1,25 @@ +"""055 Update user and activity_detail + +Revision ID: 048f7db947bf +Revises: da21b38da4db +Create Date: 2018-09-04 18:49:07.896968 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '048f7db947bf' +down_revision = 'da21b38da4db' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column('activity_detail', 'activity_id', nullable=True) + op.alter_column('user', 'name', nullable=False) + + +def downgrade(): + op.alter_column('activity_detail', 'activity_id', nullable=False) + op.alter_column('user', 'name', nullable=True) diff --git a/ckan/migration/versions/055_update_user_and_activity_detail.py b/ckan/migration/versions/055_update_user_and_activity_detail.py deleted file mode 100644 index 7e91c2038ee..00000000000 --- a/ckan/migration/versions/055_update_user_and_activity_detail.py +++ /dev/null @@ -1,11 +0,0 @@ -# encoding: utf-8 - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE activity_detail - ALTER COLUMN activity_id DROP NOT NULL; - - ALTER TABLE "user" - ALTER COLUMN name SET NOT NULL; - ''' - ) diff --git a/ckan/migration/versions/056_11af3215ae89_add_related_table.py b/ckan/migration/versions/056_11af3215ae89_add_related_table.py new file mode 100644 index 00000000000..ce048574c9d --- /dev/null +++ b/ckan/migration/versions/056_11af3215ae89_add_related_table.py @@ -0,0 +1,49 @@ +"""056 Add related table + +Revision ID: 11af3215ae89 +Revises: 048f7db947bf +Create Date: 2018-09-04 18:49:08.239860 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '11af3215ae89' +down_revision = '048f7db947bf' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'related', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('type', sa.UnicodeText, nullable=False), + sa.Column('title', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column('image_url', sa.UnicodeText), + sa.Column('url', sa.UnicodeText), sa.Column('created', sa.TIMESTAMP), + sa.Column('owner_id', sa.UnicodeText) + ) + op.create_table( + 'related_dataset', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('dataset_id', sa.UnicodeText, nullable=False), + sa.Column('related_id', sa.UnicodeText, nullable=False), + sa.Column('status', sa.UnicodeText) + ) + op.create_primary_key('related_pkey', 'related', ['id']) + op.create_primary_key('related_dataset_pkey', 'related_dataset', ['id']) + + op.create_foreign_key( + 'related_dataset_dataset_id_fkey', 'related_dataset', 'package', + ['dataset_id'], ['id'] + ) + op.create_foreign_key( + 'related_dataset_related_id_fkey', 'related_dataset', 'related', + ['related_id'], ['id'] + ) + + +def downgrade(): + op.drop_table('related_dataset') + op.drop_table('related') diff --git a/ckan/migration/versions/056_add_related_table.py b/ckan/migration/versions/056_add_related_table.py deleted file mode 100644 index f525d38cadd..00000000000 --- a/ckan/migration/versions/056_add_related_table.py +++ /dev/null @@ -1,42 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -BEGIN; -CREATE TABLE related ( - id text NOT NULL, - type text NOT NULL, - title text, - description text, - image_url text, - url text, - created timestamp without time zone, - owner_id text -); - -CREATE TABLE related_dataset ( - id text NOT NULL, - dataset_id text NOT NULL, - related_id text NOT NULL, - status text -); - -ALTER TABLE related - ADD CONSTRAINT related_pkey PRIMARY KEY (id); - -ALTER TABLE related_dataset - ADD CONSTRAINT related_dataset_pkey PRIMARY KEY (id); - -ALTER TABLE related_dataset - ADD CONSTRAINT related_dataset_dataset_id_fkey FOREIGN KEY (dataset_id) REFERENCES package(id); - -ALTER TABLE related_dataset - ADD CONSTRAINT related_dataset_related_id_fkey FOREIGN KEY (related_id) REFERENCES related(id); -COMMIT; - ''' - ) diff --git a/ckan/migration/versions/057_660a5aae527e_tracking.py b/ckan/migration/versions/057_660a5aae527e_tracking.py new file mode 100644 index 00000000000..9cadfe72fcc --- /dev/null +++ b/ckan/migration/versions/057_660a5aae527e_tracking.py @@ -0,0 +1,59 @@ +"""057 Tracking + +Revision ID: 660a5aae527e +Revises: 11af3215ae89 +Create Date: 2018-09-04 18:49:08.573692 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '660a5aae527e' +down_revision = '11af3215ae89' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'tracking_raw', sa.Column('user_key', sa.String(100), nullable=False), + sa.Column('url', sa.UnicodeText, nullable=False), + sa.Column('tracking_type', sa.String(10), nullable=False), + sa.Column( + 'access_timestamp', + sa.TIMESTAMP, + server_default=sa.func.current_timestamp() + ) + ) + op.create_index('tracking_raw_url', 'tracking_raw', ['url']) + op.create_index('tracking_raw_user_key', 'tracking_raw', ['user_key']) + op.create_index( + 'tracking_raw_access_timestamp', 'tracking_raw', ['access_timestamp'] + ) + + op.create_table( + 'tracking_summary', sa.Column('url', sa.UnicodeText, nullable=False), + sa.Column('package_id', sa.UnicodeText), + sa.Column('tracking_type', sa.String(10), nullable=False), + sa.Column('count', sa.Integer, nullable=False), + sa.Column( + 'running_total', sa.Integer, nullable=False, server_default='0' + ), + sa.Column( + 'recent_views', sa.Integer, nullable=False, server_default='0' + ), sa.Column('tracking_date', sa.Date) + ) + + op.create_index('tracking_summary_url', 'tracking_summary', ['url']) + op.create_index( + 'tracking_summary_package_id', 'tracking_summary', ['package_id'] + ) + op.create_index( + 'tracking_summary_date', 'tracking_summary', ['tracking_date'] + ) + + +def downgrade(): + op.drop_table('tracking_summary') + op.drop_table('tracking_raw') diff --git a/ckan/migration/versions/057_tracking.py b/ckan/migration/versions/057_tracking.py deleted file mode 100644 index 9c151a177be..00000000000 --- a/ckan/migration/versions/057_tracking.py +++ /dev/null @@ -1,35 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - BEGIN; - CREATE TABLE tracking_raw ( - user_key character varying(100) NOT NULL, - url text NOT NULL, - tracking_type character varying(10) NOT NULL, - access_timestamp timestamp without time zone DEFAULT current_timestamp - ); - CREATE INDEX tracking_raw_url ON tracking_raw(url); - CREATE INDEX tracking_raw_user_key ON tracking_raw(user_key); - CREATE INDEX tracking_raw_access_timestamp ON tracking_raw(access_timestamp); - - CREATE TABLE tracking_summary( - url text NOT NULL, - package_id text, - tracking_type character varying(10) NOT NULL, - count int NOT NULL, - running_total int NOT NULL DEFAULT 0, - recent_views int NOT NULL DEFAULT 0, - tracking_date date - ); - - CREATE INDEX tracking_summary_url ON tracking_summary(url); - CREATE INDEX tracking_summary_package_id ON tracking_summary(package_id); - CREATE INDEX tracking_summary_date ON tracking_summary(tracking_date); - - COMMIT; - ''' - ) diff --git a/ckan/migration/versions/058_add_follower_tables.py b/ckan/migration/versions/058_add_follower_tables.py deleted file mode 100644 index 4ed829cc697..00000000000 --- a/ckan/migration/versions/058_add_follower_tables.py +++ /dev/null @@ -1,39 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -CREATE TABLE user_following_dataset ( - follower_id text NOT NULL, - object_id text NOT NULL, - datetime timestamp without time zone NOT NULL -); - -CREATE TABLE user_following_user ( - follower_id text NOT NULL, - object_id text NOT NULL, - datetime timestamp without time zone NOT NULL -); - -ALTER TABLE user_following_dataset - ADD CONSTRAINT user_following_dataset_pkey PRIMARY KEY (follower_id, object_id); - -ALTER TABLE user_following_user - ADD CONSTRAINT user_following_user_pkey PRIMARY KEY (follower_id, object_id); - -ALTER TABLE user_following_dataset - ADD CONSTRAINT user_following_dataset_follower_id_fkey FOREIGN KEY (follower_id) REFERENCES "user"(id) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE user_following_dataset - ADD CONSTRAINT user_following_dataset_object_id_fkey FOREIGN KEY (object_id) REFERENCES package(id) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE user_following_user - ADD CONSTRAINT user_following_user_follower_id_fkey FOREIGN KEY (follower_id) REFERENCES "user"(id) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE user_following_user - ADD CONSTRAINT user_following_user_object_id_fkey FOREIGN KEY (object_id) REFERENCES "user"(id) ON UPDATE CASCADE ON DELETE CASCADE; - ''') diff --git a/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py b/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py new file mode 100644 index 00000000000..a1f5a28742e --- /dev/null +++ b/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py @@ -0,0 +1,73 @@ +"""058 Add follower tables + +Revision ID: bd36d1826a5d +Revises: 660a5aae527e +Create Date: 2018-09-04 18:49:08.908624 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'bd36d1826a5d' +down_revision = '660a5aae527e' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'user_following_dataset', + sa.Column('follower_id', sa.UnicodeText, nullable=False), + sa.Column('object_id', sa.UnicodeText, nullable=False), + sa.Column('datetime', sa.TIMESTAMP, nullable=False) + ) + op.create_table( + 'user_following_user', + sa.Column('follower_id', sa.UnicodeText, nullable=False), + sa.Column('object_id', sa.UnicodeText, nullable=False), + sa.Column('datetime', sa.TIMESTAMP, nullable=False) + ) + + op.create_primary_key( + 'user_following_dataset_pkey', 'user_following_dataset', + ['follower_id', 'object_id'] + ) + op.create_primary_key( + 'user_following_user_pkey', 'user_following_user', + ['follower_id', 'object_id'] + ) + + op.create_foreign_key( + 'user_following_dataset_follower_id_fkey', + 'user_following_dataset', + 'user', ['follower_id'], ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + op.create_foreign_key( + 'user_following_dataset_object_id_fkey', + 'user_following_dataset', + 'package', ['object_id'], ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + op.create_foreign_key( + 'user_following_user_follower_id_fkey', + 'user_following_user', + 'user', ['follower_id'], ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + op.create_foreign_key( + 'user_following_user_object_id_fkey', + 'user_following_user', + 'user', ['object_id'], ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + + +def downgrade(): + op.drop_table('user_following_dataset') + op.drop_table('user_following_user') diff --git a/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py b/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py new file mode 100644 index 00000000000..8371a37a0c7 --- /dev/null +++ b/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py @@ -0,0 +1,33 @@ +"""059 Add related count and_flag + +Revision ID: 9291bb46f352 +Revises: bd36d1826a5d +Create Date: 2018-09-04 18:49:09.243738 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '9291bb46f352' +down_revision = 'bd36d1826a5d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'related', + sa.Column( + 'view_count', sa.Integer, nullable=False, server_default='0' + ) + ) + op.add_column( + 'related', + sa.Column('featured', sa.Integer, nullable=False, server_default='0') + ) + + +def downgrade(): + op.drop_column('related', 'view_count') + op.drop_column('related', 'featured') diff --git a/ckan/migration/versions/059_add_related_count_and_flag.py b/ckan/migration/versions/059_add_related_count_and_flag.py deleted file mode 100644 index 294e5c45a12..00000000000 --- a/ckan/migration/versions/059_add_related_count_and_flag.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE "related" - ADD COLUMN view_count INT NOT NULL DEFAULT 0; - - ALTER TABLE "related" - ADD COLUMN featured INT NOT NULL DEFAULT 0; - - UPDATE related SET view_count=0, featured=0 WHERE view_count IS NULL; - ''' - ) diff --git a/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py b/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py new file mode 100644 index 00000000000..5639f9302da --- /dev/null +++ b/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py @@ -0,0 +1,47 @@ +"""060 Add system info table + +Revision ID: 31ad11c518fc +Revises: 9291bb46f352 +Create Date: 2018-09-04 18:49:09.587220 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '31ad11c518fc' +down_revision = '9291bb46f352' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.create_table( + 'system_info', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('key', sa.Unicode(100), unique=True, nullable=False), + sa.Column('value', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, sa.ForeignKey('revision.id')) + ) + + op.create_table( + 'system_info_revision', + sa.Column('id', sa.Integer, primary_key=True, nullable=False), + sa.Column('key', sa.Unicode(100), unique=True, nullable=False), + sa.Column('value', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.Integer, sa.ForeignKey('system_info.id') + ), + ) + + +def downgrade(): + op.drop_table('system_info_revision') + op.drop_table('system_info') diff --git a/ckan/migration/versions/060_add_system_info_table.py b/ckan/migration/versions/060_add_system_info_table.py deleted file mode 100644 index ceeb821036e..00000000000 --- a/ckan/migration/versions/060_add_system_info_table.py +++ /dev/null @@ -1,29 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - meta = MetaData() - - - system_info_table = Table('system_info', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('key', Unicode(100), unique=True, nullable=False), - Column('value', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id')) - ) - - system_info_revision_table = Table('system_info_revision', meta, - Column('id', Integer() , primary_key=True, nullable=False), - Column('key', Unicode(100), unique=True, nullable=False), - Column('value', UnicodeText), - Column('revision_id', UnicodeText, ForeignKey('revision.id'), primary_key=True), - Column('continuity_id', Integer(), ForeignKey('system_info.id') ), - ) - - - meta.bind = migrate_engine - revision_table = Table('revision', meta, autoload=True) - - meta.create_all() diff --git a/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py b/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py new file mode 100644 index 00000000000..a3a9185fa5a --- /dev/null +++ b/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py @@ -0,0 +1,49 @@ +"""061 Add follower group_table + +Revision ID: 338d460bc460 +Revises: 31ad11c518fc +Create Date: 2018-09-04 18:49:09.925977 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '338d460bc460' +down_revision = '31ad11c518fc' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'user_following_group', + sa.Column('follower_id', sa.UnicodeText, nullable=False), + sa.Column('object_id', sa.UnicodeText, nullable=False), + sa.Column('datetime', sa.TIMESTAMP, nullable=False) + ) + + op.create_primary_key( + 'user_following_group_pkey', 'user_following_group', + ['follower_id', 'object_id'] + ) + op.create_foreign_key( + 'user_following_group_user_id_fkey', + 'user_following_group', + 'user', ['follower_id'], + ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + op.create_foreign_key( + 'user_following_group_group_id_fkey', + 'user_following_group', + 'group', ['object_id'], + ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + + +def downgrade(): + op.drop_table('user_following_group') diff --git a/ckan/migration/versions/061_add_follower__group_table.py b/ckan/migration/versions/061_add_follower__group_table.py deleted file mode 100644 index 5fe27e2fc9e..00000000000 --- a/ckan/migration/versions/061_add_follower__group_table.py +++ /dev/null @@ -1,24 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -CREATE TABLE user_following_group ( - follower_id text NOT NULL, - object_id text NOT NULL, - datetime timestamp without time zone NOT NULL -); - -ALTER TABLE user_following_group - ADD CONSTRAINT user_following_group_pkey PRIMARY KEY (follower_id, object_id); - -ALTER TABLE user_following_group - ADD CONSTRAINT user_following_group_user_id_fkey FOREIGN KEY (follower_id) REFERENCES "user"(id) ON UPDATE CASCADE ON DELETE CASCADE; - -ALTER TABLE user_following_group - ADD CONSTRAINT user_following_group_group_id_fkey FOREIGN KEY (object_id) REFERENCES "group"(id) ON UPDATE CASCADE ON DELETE CASCADE; - ''') diff --git a/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py b/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py new file mode 100644 index 00000000000..8d3e063b17c --- /dev/null +++ b/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py @@ -0,0 +1,34 @@ +"""062 Add dashboard table + +Revision ID: 6deb2bbab394 +Revises: 338d460bc460 +Create Date: 2018-09-04 18:49:10.266290 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '6deb2bbab394' +down_revision = '338d460bc460' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'dashboard', sa.Column('user_id', sa.UnicodeText, nullable=False), + sa.Column('activity_stream_last_viewed', sa.TIMESTAMP, nullable=False) + ) + op.create_primary_key('dashboard_pkey', 'dashboard', ['user_id']) + op.create_foreign_key( + 'dashboard_user_id_fkey', + 'dashboard', + 'user', ['user_id'], ['id'], + onupdate='CASCADE', + ondelete='CASCADE' + ) + + +def downgrade(): + op.drop_table('dashboard') diff --git a/ckan/migration/versions/062_add_dashboard_table.py b/ckan/migration/versions/062_add_dashboard_table.py deleted file mode 100644 index c820ba372d0..00000000000 --- a/ckan/migration/versions/062_add_dashboard_table.py +++ /dev/null @@ -1,18 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -CREATE TABLE dashboard ( - user_id text NOT NULL, - activity_stream_last_viewed timestamp without time zone NOT NULL -); -ALTER TABLE dashboard - ADD CONSTRAINT dashboard_pkey PRIMARY KEY (user_id); -ALTER TABLE dashboard - ADD CONSTRAINT dashboard_user_id_fkey FOREIGN KEY (user_id) REFERENCES "user"(id) ON UPDATE CASCADE ON DELETE CASCADE; - ''') diff --git a/ckan/migration/versions/063_8b633852cb7a_org_changes.py b/ckan/migration/versions/063_8b633852cb7a_org_changes.py new file mode 100644 index 00000000000..3569ccbc35b --- /dev/null +++ b/ckan/migration/versions/063_8b633852cb7a_org_changes.py @@ -0,0 +1,54 @@ +"""063 Org changes + +Revision ID: 8b633852cb7a +Revises: 6deb2bbab394 +Create Date: 2018-09-04 18:49:10.608831 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '8b633852cb7a' +down_revision = '6deb2bbab394' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.add_column( + 'user', sa.Column('sysadmin', sa.Boolean, server_default='FALSE') + ) + op.add_column('package', sa.Column('owner_org', sa.UnicodeText)) + op.add_column( + 'package', sa.Column('private', sa.Boolean, server_default='FALSE') + ) + + op.add_column('package_revision', sa.Column('owner_org', sa.UnicodeText)) + op.add_column( + 'package_revision', + sa.Column('private', sa.Boolean, server_default='FALSE') + ) + + op.add_column( + 'group', + sa.Column('is_organization', sa.Boolean, server_default='FALSE') + ) + op.add_column( + 'group_revision', + sa.Column('is_organization', sa.Boolean, server_default='FALSE') + ) + + +def downgrade(): + + op.drop_column('user', 'sysadmin') + op.drop_column('package', 'owner_org') + op.drop_column('package', 'private') + + op.drop_column('package_revision', 'owner_org') + op.drop_column('package_revision', 'private') + + op.drop_column('group', 'is_organization') + op.drop_column('group_revision', 'is_organization') diff --git a/ckan/migration/versions/063_org_changes.py b/ckan/migration/versions/063_org_changes.py deleted file mode 100644 index d2ce6772ad5..00000000000 --- a/ckan/migration/versions/063_org_changes.py +++ /dev/null @@ -1,33 +0,0 @@ -# encoding: utf-8 - -from migrate import * - -def upgrade(migrate_engine): - - update_schema = ''' -BEGIN; - -ALTER TABLE "user" - ADD COLUMN sysadmin boolean DEFAULT FALSE; - -ALTER TABLE package - ADD COLUMN owner_org TEXT, - ADD COLUMN private boolean DEFAULT FALSE; - -ALTER TABLE package_revision - ADD COLUMN owner_org TEXT, - ADD COLUMN private boolean DEFAULT FALSE; - - -ALTER TABLE "group" - ADD COLUMN is_organization boolean DEFAULT FALSE; - -ALTER TABLE group_revision - ADD COLUMN is_organization boolean DEFAULT FALSE; - -UPDATE "user" SET sysadmin=true WHERE id in ( SELECT user_id FROM user_object_role WHERE role='admin' AND context='System'); - -COMMIT; - -''' - migrate_engine.execute(update_schema) diff --git a/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py b/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py new file mode 100644 index 00000000000..96c571df8f4 --- /dev/null +++ b/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py @@ -0,0 +1,31 @@ +"""064 Add email_last_sent_column + +Revision ID: 4f8becd4919a +Revises: 8b633852cb7a +Create Date: 2018-09-04 18:49:10.939639 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '4f8becd4919a' +down_revision = '8b633852cb7a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'dashboard', + sa.Column( + 'email_last_sent', + sa.TIMESTAMP, + nullable=False, + server_default=sa.func.localtimestamp() + ) + ) + + +def downgrade(): + op.drop_column('dashboard', 'email_last_sent') diff --git a/ckan/migration/versions/064_add_email_last_sent_column.py b/ckan/migration/versions/064_add_email_last_sent_column.py deleted file mode 100644 index d47a74cf9f6..00000000000 --- a/ckan/migration/versions/064_add_email_last_sent_column.py +++ /dev/null @@ -1,12 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -ALTER TABLE dashboard - ADD COLUMN email_last_sent timestamp without time zone NOT NULL DEFAULT LOCALTIMESTAMP; - ''') diff --git a/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py b/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py new file mode 100644 index 00000000000..aa594e63be8 --- /dev/null +++ b/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py @@ -0,0 +1,30 @@ +"""065 Add email notifications preference + +Revision ID: 1fab0bc6439e +Revises: 4f8becd4919a +Create Date: 2018-09-04 18:49:11.283832 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '1fab0bc6439e' +down_revision = '4f8becd4919a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'user', + sa.Column( + 'activity_streams_email_notifications', + sa.Boolean, + server_default='FALSE' + ) + ) + + +def downgrade(): + op.drop_column('user', 'activity_streams_email_notifications') diff --git a/ckan/migration/versions/065_add_email_notifications_preference.py b/ckan/migration/versions/065_add_email_notifications_preference.py deleted file mode 100644 index ced562eb589..00000000000 --- a/ckan/migration/versions/065_add_email_notifications_preference.py +++ /dev/null @@ -1,12 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import * -from migrate import * - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -ALTER TABLE public.user - ADD COLUMN activity_streams_email_notifications BOOLEAN DEFAULT FALSE; - ''') diff --git a/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py b/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py new file mode 100644 index 00000000000..dd56a043af1 --- /dev/null +++ b/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py @@ -0,0 +1,23 @@ +"""066 Default package type + +Revision ID: ad16b3bd8cb6 +Revises: 1fab0bc6439e +Create Date: 2018-09-04 18:49:11.624055 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'ad16b3bd8cb6' +down_revision = '1fab0bc6439e' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/066_default_package_type.py b/ckan/migration/versions/066_default_package_type.py deleted file mode 100644 index 48ccec0153d..00000000000 --- a/ckan/migration/versions/066_default_package_type.py +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - - update_statement = ''' -BEGIN; - -UPDATE package SET type = 'dataset' WHERE type IS NULL; - -COMMIT; - -''' - migrate_engine.execute(update_statement) diff --git a/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py b/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py new file mode 100644 index 00000000000..edbc2cdc496 --- /dev/null +++ b/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py @@ -0,0 +1,23 @@ +"""067 Turn extras to strings + +Revision ID: 266c110eafec +Revises: ad16b3bd8cb6 +Create Date: 2018-09-04 18:49:11.961287 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '266c110eafec' +down_revision = 'ad16b3bd8cb6' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/067_turn_extras_to_strings.py b/ckan/migration/versions/067_turn_extras_to_strings.py deleted file mode 100644 index c69be512969..00000000000 --- a/ckan/migration/versions/067_turn_extras_to_strings.py +++ /dev/null @@ -1,30 +0,0 @@ -# encoding: utf-8 - -import json - -def upgrade(migrate_engine): - - with migrate_engine.begin() as connection: - tables = 'package_extra group_extra' - revision_tables = 'package_extra_revision group_extra_revision' - - for table in tables.split(): - sql = """select id, value from {table} where substr(value,1,1) = '"' """.format(table=table) - results = connection.execute(sql) - for result in results: - id, value = result - update_sql = 'update {table} set value = %s where id = %s' - connection.execute(update_sql.format(table=table), - json.loads(value), id) - - for table in revision_tables.split(): - sql = """select id, revision_id, value from {table} where substr(value,1,1) = '"' """.format(table=table) - - results = connection.execute(sql) - for result in results: - id, revision_id, value = result - update_sql = 'update {table} set value = %s where id = %s and revision_id = %s' - connection.execute(update_sql.format(table=table), - json.loads(value), id, revision_id) - - diff --git a/ckan/migration/versions/068_add_package_extras_index.py b/ckan/migration/versions/068_add_package_extras_index.py deleted file mode 100644 index 1aac25089e5..00000000000 --- a/ckan/migration/versions/068_add_package_extras_index.py +++ /dev/null @@ -1,10 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - CREATE INDEX idx_package_extra_package_id ON package_extra_revision - USING BTREE (package_id, current); - ''' - ) diff --git a/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py b/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py new file mode 100644 index 00000000000..d6f27a06268 --- /dev/null +++ b/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py @@ -0,0 +1,26 @@ +"""068 Add package extras index + +Revision ID: e33a5f2b2a84 +Revises: 266c110eafec +Create Date: 2018-09-04 18:49:12.301432 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e33a5f2b2a84' +down_revision = '266c110eafec' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + 'idx_package_extra_package_id', 'package_extra_revision', + ['package_id', 'current'] + ) + + +def downgrade(): + op.drop_index('idx_package_extra_package_id', 'package_extra_revision') diff --git a/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py b/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py new file mode 100644 index 00000000000..1137d615f92 --- /dev/null +++ b/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py @@ -0,0 +1,41 @@ +"""069 Resource url and metadata_modified + +Revision ID: e7524c675cdb +Revises: e33a5f2b2a84 +Create Date: 2018-09-04 18:49:12.639912 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e7524c675cdb' +down_revision = 'e33a5f2b2a84' +branch_labels = None +depends_on = None + + +def upgrade(): + + op.add_column('resource', sa.Column('url_type', sa.UnicodeText)) + op.add_column('resource_revision', sa.Column('url_type', sa.UnicodeText)) + + op.add_column('package', sa.Column('metadata_modified', sa.TIMESTAMP)) + op.add_column('package', sa.Column('creator_user_id', sa.UnicodeText)) + op.add_column( + 'package_revision', sa.Column('metadata_modified', sa.TIMESTAMP) + ) + op.add_column( + 'package_revision', sa.Column('creator_user_id', sa.UnicodeText) + ) + + +def downgrade(): + + op.drop_column('resource', 'url_type') + op.drop_column('resource_revision', 'url_type') + + op.drop_column('package', 'metadata_modified') + op.drop_column('package', 'creator_user_id') + op.drop_column('package_revision', 'metadata_modified') + op.drop_column('package_revision', 'creator_user_id') diff --git a/ckan/migration/versions/069_resource_url_and_metadata_modified.py b/ckan/migration/versions/069_resource_url_and_metadata_modified.py deleted file mode 100644 index 888d146f93c..00000000000 --- a/ckan/migration/versions/069_resource_url_and_metadata_modified.py +++ /dev/null @@ -1,76 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - - update_schema = ''' -BEGIN; - -ALTER TABLE resource - ADD COLUMN url_type TEXT; - -ALTER TABLE resource_revision - ADD COLUMN url_type TEXT; - -ALTER TABLE package - ADD COLUMN metadata_modified timestamp without time zone, - ADD COLUMN creator_user_id TEXT; - -ALTER TABLE package_revision - ADD COLUMN metadata_modified timestamp without time zone, - ADD COLUMN creator_user_id TEXT; - - --- package -update package -set metadata_modified = greatest(max_revision, metadata_modified) -from -( -select id package_id, max(revision_timestamp) max_revision -from package_revision group by id -) max_rev -where max_rev.package_id = package.id; - --- package tag -update package -set metadata_modified = greatest(max_revision, metadata_modified) -from (select package_id, max(revision_timestamp) max_revision -from package_tag_revision group by package_id) max_rev -where max_rev.package_id = package.id; - --- package extra -update package -set metadata_modified = greatest(max_revision, metadata_modified) -from (select package_id, max(revision_timestamp) max_revision -from package_extra_revision group by package_id) max_rev -where max_rev.package_id = package.id; - ---resource - -update package -set metadata_modified = greatest(max_revision, metadata_modified) -from (select package_id, max(revision_timestamp) max_revision - from resource_revision - join resource_group - on resource_revision.resource_group_id = resource_group.id - group by package_id) max_rev -where max_rev.package_id = package.id -; - --- add as many creators as we can find -update package set creator_user_id = user_id from -(select - package_revision.id as package_id, - "user".id as user_id, revision_timestamp, - row_number() over - (partition by package_revision.id order by revision_timestamp) num -from package_revision - join revision on package_revision.revision_id = revision.id join "user" - on (revision.author = "user".name - or revision.author = "user".openid)) first_rev - where package_id = id and num = 1; - -COMMIT; - -''' - migrate_engine.execute(update_schema) diff --git a/ckan/migration/versions/070_add_activity_and_resource_indexes.py b/ckan/migration/versions/070_add_activity_and_resource_indexes.py deleted file mode 100644 index 62c1a3802e0..00000000000 --- a/ckan/migration/versions/070_add_activity_and_resource_indexes.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - CREATE INDEX idx_activity_user_id ON activity - (user_id, timestamp); - CREATE INDEX idx_activity_object_id ON activity - (object_id, timestamp); - CREATE INDEX idx_activity_detail_activity_id ON activity_detail - (activity_id); - CREATE INDEX idx_resource_resource_group_id ON resource_revision - (resource_group_id, current); - ''' - ) diff --git a/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py b/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py new file mode 100644 index 00000000000..3498045a108 --- /dev/null +++ b/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py @@ -0,0 +1,38 @@ +"""070 Add activity and resource_indexes + +Revision ID: cfb544112fa7 +Revises: e7524c675cdb +Create Date: 2018-09-04 18:49:13.010411 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'cfb544112fa7' +down_revision = 'e7524c675cdb' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + 'idx_activity_user_id', 'activity', ['user_id', 'timestamp'] + ) + op.create_index( + 'idx_activity_object_id', 'activity', ['object_id', 'timestamp'] + ) + op.create_index( + 'idx_activity_detail_activity_id', 'activity_detail', ['activity_id'] + ) + op.create_index( + 'idx_resource_resource_group_id', 'resource_revision', + ['resource_group_id', 'current'] + ) + + +def downgrade(): + op.drop_index('idx_activity_user_id', 'activity') + op.drop_index('idx_activity_object_id', 'activity') + op.drop_index('idx_activity_detail_activity_id', 'activity_detail') + op.drop_index('idx_resource_resource_group_id', 'resource_revision') diff --git a/ckan/migration/versions/071_add_state_column_to_user_table.py b/ckan/migration/versions/071_add_state_column_to_user_table.py deleted file mode 100644 index e890b5b05be..00000000000 --- a/ckan/migration/versions/071_add_state_column_to_user_table.py +++ /dev/null @@ -1,11 +0,0 @@ -# encoding: utf-8 - -import ckan.model - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - ALTER TABLE "user" ADD COLUMN "state" text NOT NULL DEFAULT '%s' - ''' % ckan.model.State.ACTIVE - ) diff --git a/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py b/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py new file mode 100644 index 00000000000..930a826b23b --- /dev/null +++ b/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py @@ -0,0 +1,28 @@ +"""071 Add state column to_user_table + +Revision ID: c16f081ef73a +Revises: cfb544112fa7 +Create Date: 2018-09-04 18:49:13.351494 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'c16f081ef73a' +down_revision = 'cfb544112fa7' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'user', + sa.Column( + 'state', sa.UnicodeText, nullable=False, server_default='active' + ) + ) + + +def downgrade(): + op.drop_column('user', 'state') diff --git a/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py b/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py new file mode 100644 index 00000000000..45d992e7821 --- /dev/null +++ b/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py @@ -0,0 +1,38 @@ +"""072 Add resource view + +Revision ID: 08dcb9233ad7 +Revises: c16f081ef73a +Create Date: 2018-09-04 18:49:13.697490 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '08dcb9233ad7' +down_revision = 'c16f081ef73a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'resource_view', + sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('resource_id', sa.UnicodeText), + sa.Column('title', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column('view_type', sa.UnicodeText, nullable=False), + sa.Column('order', sa.Integer, nullable=False), + sa.Column('config', sa.UnicodeText), + ) + + op.create_primary_key('resource_view_pkey', 'resource_view', ['id']) + op.create_foreign_key( + 'resource_view_resource_id_fkey', 'resource_view', 'resource', + ['resource_id'], ['id'] + ) + + +def downgrade(): + op.drop_table('resource_view') diff --git a/ckan/migration/versions/072_add_resource_view.py b/ckan/migration/versions/072_add_resource_view.py deleted file mode 100644 index 57d22236a1c..00000000000 --- a/ckan/migration/versions/072_add_resource_view.py +++ /dev/null @@ -1,26 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute(''' - BEGIN; - - CREATE TABLE resource_view ( - id text NOT NULL, - resource_id text, - title text, - description text, - view_type text NOT NULL, - "order" integer NOT NULL, - config text - ); - - ALTER TABLE resource_view - ADD CONSTRAINT resource_view_pkey PRIMARY KEY (id); - - ALTER TABLE resource_view - ADD CONSTRAINT resource_view_resource_id_fkey - FOREIGN KEY (resource_id) REFERENCES resource(id); - - COMMIT; - ''') diff --git a/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py b/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py new file mode 100644 index 00000000000..b8770af40b1 --- /dev/null +++ b/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py @@ -0,0 +1,34 @@ +"""073 Update resource view resource_id_constraint + +Revision ID: 011f51208be3 +Revises: 08dcb9233ad7 +Create Date: 2018-09-04 18:49:14.072410 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '011f51208be3' +down_revision = '08dcb9233ad7' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint('resource_view_resource_id_fkey', 'resource_view') + op.create_foreign_key( + 'resource_view_resource_id_fkey', + 'resource_view', + 'resource', ['resource_id'], ['id'], + ondelete='CASCADE', + onupdate='CASCADE' + ) + + +def downgrade(): + op.drop_constraint('resource_view_resource_id_fkey', 'resource_view') + op.create_foreign_key( + 'resource_view_resource_id_fkey', 'resource_view', 'resource', + ['resource_id'], ['id'] + ) diff --git a/ckan/migration/versions/073_update_resource_view_resource_id_constraint.py b/ckan/migration/versions/073_update_resource_view_resource_id_constraint.py deleted file mode 100644 index 6a6d4e79cb7..00000000000 --- a/ckan/migration/versions/073_update_resource_view_resource_id_constraint.py +++ /dev/null @@ -1,17 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute(''' - BEGIN; - - ALTER TABLE resource_view - DROP CONSTRAINT resource_view_resource_id_fkey; - - ALTER TABLE resource_view - ADD CONSTRAINT resource_view_resource_id_fkey - FOREIGN KEY (resource_id) REFERENCES resource(id) - ON UPDATE CASCADE ON DELETE CASCADE; - - COMMIT; - ''') diff --git a/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py b/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py new file mode 100644 index 00000000000..07dc9107fd3 --- /dev/null +++ b/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py @@ -0,0 +1,128 @@ +"""074 Remove resource groups + +Revision ID: a4ca55f0f45e +Revises: 011f51208be3 +Create Date: 2018-09-04 18:49:14.423978 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'a4ca55f0f45e' +down_revision = '011f51208be3' +branch_labels = None +depends_on = None + +resource_indexes = (( + 'resource_revision', (( + 'idx_resource_period_resource_group', + ('revision_timestamp', 'expired_timestamp', 'resource_group_id') + ), ) +), ( + 'resource_group_revision', ( + ( + 'idx_resource_group_period', + ('revision_timestamp', 'expired_timestamp', 'id') + ), + ( + 'idx_resource_group_period_package', + ('revision_timestamp', 'expired_timestamp', 'package_id') + ), + ('idx_resource_group_current', ('current', )), + ) +)) + + +def upgrade(): + op.add_column( + 'resource', + sa.Column( + 'package_id', sa.UnicodeText, nullable=False, server_default='' + ) + ) + op.drop_column('resource', 'resource_group_id') + op.add_column( + 'resource_revision', + sa.Column( + 'package_id', sa.UnicodeText, nullable=False, server_default='' + ) + ) + op.drop_column('resource_revision', 'resource_group_id') + + op.drop_table('resource_group_revision') + op.drop_table('resource_group') + + +def downgrade(): + op.create_table( + 'resource_group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + sa.Column('label', sa.UnicodeText), + sa.Column('sort_order', sa.UnicodeText), + sa.Column('extras', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column('revision_id', sa.UnicodeText, sa.ForeignKey('revision.id')), + ) + + op.create_table( + 'resource_group_revision', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + sa.Column('label', sa.UnicodeText), + sa.Column('sort_order', sa.UnicodeText), + sa.Column('extras', sa.UnicodeText), + sa.Column('state', sa.UnicodeText), + sa.Column( + 'revision_id', + sa.UnicodeText, + sa.ForeignKey('revision.id'), + primary_key=True + ), + sa.Column( + 'continuity_id', sa.UnicodeText, + sa.ForeignKey('resource_group.id') + ), sa.Column('expired_id', sa.UnicodeText), + sa.Column('revision_timestamp', sa.TIMESTAMP), + sa.Column('expired_timestamp', sa.TIMESTAMP), + sa.Column('current', sa.Boolean) + ) + + op.drop_column('resource', 'package_id') + op.drop_column('resource_revision', 'package_id') + + op.add_column( + 'resource', + sa.Column( + 'resource_group_id', + sa.UnicodeText, + nullable=False, + server_default='' + ) + ) + op.add_column( + 'resource_revision', + sa.Column( + 'resource_group_id', + sa.UnicodeText, + nullable=False, + server_default='' + ) + ) + op.create_index( + 'idx_resource_resource_group_id', 'resource_revision', + ['resource_group_id', 'current'] + ) + + for table, indexes in resource_indexes: + for index, fields in indexes: + op.create_index(index, table, fields) + op.create_foreign_key( + 'resource_resource_group_id_fkey', 'resource', 'resource_group', + ['resource_group_id'], ['id'] + ) + op.create_foreign_key( + 'resource_revision_resource_group_id_fkey', 'resource_revision', + 'resource_group', ['resource_group_id'], ['id'] + ) diff --git a/ckan/migration/versions/074_remove_resource_groups.py b/ckan/migration/versions/074_remove_resource_groups.py deleted file mode 100644 index 9a55dd192ec..00000000000 --- a/ckan/migration/versions/074_remove_resource_groups.py +++ /dev/null @@ -1,31 +0,0 @@ -# encoding: utf-8 - -import ckan.model - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - ALTER TABLE "resource" - ADD COLUMN "package_id" text NOT NULL DEFAULT ''; - UPDATE "resource" AS R - SET package_id = G.package_id - FROM "resource_group" AS G - WHERE R.resource_group_id = G.id; - ALTER TABLE "resource" DROP COLUMN "resource_group_id"; - - ALTER TABLE "resource_revision" - ADD COLUMN "package_id" text NOT NULL DEFAULT ''; - UPDATE "resource_revision" AS R - SET package_id = G.package_id - FROM "resource_group_revision" AS G - WHERE R.resource_group_id = G.id; - ALTER TABLE "resource_revision" DROP COLUMN "resource_group_id"; - - ALTER TABLE resource_group_revision - DROP CONSTRAINT resource_group_revision_continuity_id_fkey; - - DROP TABLE "resource_group_revision"; - DROP TABLE "resource_group"; - ''' - ) diff --git a/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py b/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py new file mode 100644 index 00000000000..3e6b766e068 --- /dev/null +++ b/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py @@ -0,0 +1,23 @@ +"""075 Rename view plugins + +Revision ID: 9cdc88c8896a +Revises: a4ca55f0f45e +Create Date: 2018-09-04 18:49:14.766120 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '9cdc88c8896a' +down_revision = 'a4ca55f0f45e' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/075_rename_view_plugins.py b/ckan/migration/versions/075_rename_view_plugins.py deleted file mode 100644 index f23c4eb4d9b..00000000000 --- a/ckan/migration/versions/075_rename_view_plugins.py +++ /dev/null @@ -1,17 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - BEGIN; - - UPDATE resource_view - SET view_type = 'image_view' WHERE view_type = 'image'; - - UPDATE resource_view - SET view_type = 'webpage_view' WHERE view_type = 'webpage'; - - COMMIT; - ''' - ) diff --git a/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py b/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py new file mode 100644 index 00000000000..c9e178da24d --- /dev/null +++ b/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py @@ -0,0 +1,23 @@ +"""076 Rename view plugins 2 + +Revision ID: 59995aa965c0 +Revises: 9cdc88c8896a +Create Date: 2018-09-04 18:49:15.123438 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '59995aa965c0' +down_revision = '9cdc88c8896a' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/076_rename_view_plugins_2.py b/ckan/migration/versions/076_rename_view_plugins_2.py deleted file mode 100644 index 278879e14c3..00000000000 --- a/ckan/migration/versions/076_rename_view_plugins_2.py +++ /dev/null @@ -1,17 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - BEGIN; - - UPDATE resource_view - SET view_type = 'text_view' WHERE view_type = 'text'; - - UPDATE resource_view - SET view_type = 'pdf_view' WHERE view_type = 'pdf'; - - COMMIT; - ''' - ) diff --git a/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py b/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py new file mode 100644 index 00000000000..bc23b1dbf04 --- /dev/null +++ b/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py @@ -0,0 +1,57 @@ +"""077 Add revisions to system_info + +Revision ID: 51171a04d86d +Revises: 59995aa965c0 +Create Date: 2018-09-04 18:49:15.478074 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '51171a04d86d' +down_revision = '59995aa965c0' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'system_info', + sa.Column( + 'state', sa.UnicodeText, nullable=False, server_default='active' + ) + ) + op.add_column( + 'system_info_revision', + sa.Column( + 'state', sa.UnicodeText, nullable=False, server_default='active' + ) + ) + + op.add_column( + 'system_info_revision', sa.Column('expired_id', sa.UnicodeText) + ) + op.add_column( + 'system_info_revision', sa.Column('revision_timestamp', sa.TIMESTAMP) + ) + op.add_column( + 'system_info_revision', sa.Column('expired_timestamp', sa.TIMESTAMP) + ) + op.add_column('system_info_revision', sa.Column('current', sa.Boolean)) + + op.drop_constraint('system_info_revision_key_key', 'system_info_revision') + + +def downgrade(): + op.create_unique_constraint( + 'system_info_revision_key_key', 'system_info_revision', ['key'] + ) + + op.drop_column('system_info', 'state') + op.drop_column('system_info_revision', 'state') + + op.drop_column('system_info_revision', 'expired_id') + op.drop_column('system_info_revision', 'revision_timestamp') + op.drop_column('system_info_revision', 'expired_timestamp') + op.drop_column('system_info_revision', 'current') diff --git a/ckan/migration/versions/077_add_revisions_to_system_info.py b/ckan/migration/versions/077_add_revisions_to_system_info.py deleted file mode 100644 index 9516a9127c0..00000000000 --- a/ckan/migration/versions/077_add_revisions_to_system_info.py +++ /dev/null @@ -1,25 +0,0 @@ -# encoding: utf-8 - -import vdm.sqlalchemy - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - ALTER TABLE "system_info" - ADD COLUMN "state" text NOT NULL DEFAULT '{state}'; - - ALTER TABLE "system_info_revision" - ADD COLUMN "state" text NOT NULL DEFAULT '{state}'; - - ALTER TABLE system_info_revision - ADD COLUMN expired_id text, - ADD COLUMN revision_timestamp timestamp without time zone, - ADD COLUMN expired_timestamp timestamp without time zone, - ADD COLUMN current boolean; - - ALTER TABLE system_info_revision - DROP CONSTRAINT "system_info_revision_key_key"; - - '''.format(state=vdm.sqlalchemy.State.ACTIVE) - ) diff --git a/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py b/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py new file mode 100644 index 00000000000..8fa573e85c0 --- /dev/null +++ b/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py @@ -0,0 +1,109 @@ +"""078 Remove old authz model + +Revision ID: ae821876532a +Revises: 51171a04d86d +Create Date: 2018-09-04 18:49:15.812926 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'ae821876532a' +down_revision = '51171a04d86d' +branch_labels = None +depends_on = None + +indexes = ( + ('idx_uor_id', 'user_object_role', ['id']), + ('idx_uor_user_id', 'user_object_role', ['user_id']), + ('idx_uor_context', 'user_object_role', ['context']), + ('idx_uor_role', 'user_object_role', ['role']), + ('idx_uor_user_id_role', 'user_object_role', ['user_id', 'role']), + ('idx_ra_role', 'role_action', ['role']), + ('idx_ra_action', 'role_action', ['action']), + ('idx_ra_role_action', 'role_action', ['action', 'role']), +) + +def upgrade(): + op.drop_table('role_action') + op.drop_table('package_role') + op.drop_table('group_role') + op.drop_table('system_role') + op.drop_table('authorization_group_role') + op.drop_table('user_object_role') + + +def downgrade(): + + op.create_table( + 'user_object_role', sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('user_id', sa.UnicodeText, sa.ForeignKey('user.id')), + sa.Column('context', sa.UnicodeText, nullable=False), + sa.Column('role', sa.UnicodeText) + ) + + op.create_table( + 'authorization_group_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column( + 'authorization_group_id', sa.UnicodeText, + sa.ForeignKey('authorization_group.id') + ) + ) + + op.create_table( + 'system_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + ) + op.create_table( + 'group_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column('group_id', sa.UnicodeText, sa.ForeignKey('group.id')), + ) + + op.create_table( + 'package_role', + sa.Column( + 'user_object_role_id', + sa.UnicodeText, + sa.ForeignKey('user_object_role.id'), + primary_key=True + ), + sa.Column('package_id', sa.UnicodeText, sa.ForeignKey('package.id')), + ) + + op.create_table( + 'role_action', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('role', sa.UnicodeText), + sa.Column('context', sa.UnicodeText, nullable=False), + sa.Column('action', sa.UnicodeText), + ) + + op.add_column( + 'user_object_role', + sa.Column( + 'authorized_group_id', + sa.UnicodeText, + sa.ForeignKey('authorization_group.id'), + nullable=True + ) + ) + for name, table, columns in indexes: + op.create_index(name, table, columns) diff --git a/ckan/migration/versions/078_remove_old_authz_model.py b/ckan/migration/versions/078_remove_old_authz_model.py deleted file mode 100644 index a67679c698d..00000000000 --- a/ckan/migration/versions/078_remove_old_authz_model.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 - -import ckan.model - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - DROP TABLE "role_action"; - DROP TABLE "package_role"; - DROP TABLE "group_role"; - DROP TABLE "system_role"; - DROP TABLE "authorization_group_role"; - DROP TABLE "user_object_role"; - ''' - ) diff --git a/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py b/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py new file mode 100644 index 00000000000..c947363382e --- /dev/null +++ b/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py @@ -0,0 +1,25 @@ +"""079 Resource revision index + +Revision ID: e0177a15d2c9 +Revises: ae821876532a +Create Date: 2018-09-04 18:49:16.198887 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'e0177a15d2c9' +down_revision = 'ae821876532a' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + 'idx_resource_continuity_id', 'resource_revision', ['continuity_id'] + ) + + +def downgrade(): + op.drop_index('idx_resource_continuity_id', 'resource_revision') diff --git a/ckan/migration/versions/079_resource_revision_index.py b/ckan/migration/versions/079_resource_revision_index.py deleted file mode 100644 index aeabea8349e..00000000000 --- a/ckan/migration/versions/079_resource_revision_index.py +++ /dev/null @@ -1,10 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - CREATE INDEX idx_resource_continuity_id - ON resource_revision (continuity_id); - ''' - ) diff --git a/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py b/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py new file mode 100644 index 00000000000..76252c2cd8e --- /dev/null +++ b/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py @@ -0,0 +1,39 @@ +"""080 Continuity id indexes + +Revision ID: 8224d872c64f +Revises: e0177a15d2c9 +Create Date: 2018-09-04 18:49:16.546952 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '8224d872c64f' +down_revision = 'e0177a15d2c9' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + 'idx_member_continuity_id', 'member_revision', ['continuity_id'] + ) + op.create_index( + 'idx_package_tag_continuity_id', 'package_tag_revision', + ['continuity_id'] + ) + op.create_index( + 'idx_package_continuity_id', 'package_revision', ['continuity_id'] + ) + op.create_index( + 'idx_package_extra_continuity_id', 'package_extra_revision', + ['continuity_id'] + ) + + +def downgrade(): + op.drop_index('idx_member_continuity_id', 'member_revision') + op.drop_index('idx_package_tag_continuity_id', 'package_tag_revision') + op.drop_index('idx_package_continuity_id', 'package_revision') + op.drop_index('idx_package_extra_continuity_id', 'package_extra_revision') diff --git a/ckan/migration/versions/080_continuity_id_indexes.py b/ckan/migration/versions/080_continuity_id_indexes.py deleted file mode 100644 index 81641957707..00000000000 --- a/ckan/migration/versions/080_continuity_id_indexes.py +++ /dev/null @@ -1,16 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - CREATE INDEX idx_member_continuity_id - ON member_revision (continuity_id); - CREATE INDEX idx_package_tag_continuity_id - ON package_tag_revision (continuity_id); - CREATE INDEX idx_package_continuity_id - ON package_revision (continuity_id); - CREATE INDEX idx_package_extra_continuity_id - ON package_extra_revision (continuity_id); - ''' - ) diff --git a/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py b/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py new file mode 100644 index 00000000000..bb0ee315ff7 --- /dev/null +++ b/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py @@ -0,0 +1,24 @@ +"""081 Set datastore active + +Revision ID: a64cf4a79182 +Revises: 8224d872c64f +Create Date: 2018-09-04 18:49:16.896531 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'a64cf4a79182' +down_revision = '8224d872c64f' +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/ckan/migration/versions/081_set_datastore_active.py b/ckan/migration/versions/081_set_datastore_active.py deleted file mode 100644 index a8219800778..00000000000 --- a/ckan/migration/versions/081_set_datastore_active.py +++ /dev/null @@ -1,67 +0,0 @@ -# encoding: utf-8 - -import json -from sqlalchemy import create_engine -from sqlalchemy.sql import text -from sqlalchemy.exc import SQLAlchemyError -from ckan.common import config - - -def upgrade(migrate_engine): - - datastore_connection_url = config.get( - 'ckan.datastore.read_url', config.get('ckan.datastore.write_url')) - - if not datastore_connection_url: - return - - try: - datastore_engine = create_engine(datastore_connection_url) - except SQLAlchemyError: - return - - try: - datastore_connection = datastore_engine.connect() - except SQLAlchemyError: - datastore_engine.dispose() - return - - try: - - resources_in_datastore = datastore_connection.execute(''' - SELECT table_name - FROM information_schema.tables - WHERE table_schema = 'public' - AND table_name != '_table_metadata' - ''') - - if resources_in_datastore.rowcount: - - resources = migrate_engine.execute(''' - SELECT id, extras - FROM resource - WHERE id IN ({0}) AND extras IS NOT NULL - '''.format( - ','.join(['\'{0}\''.format(_id[0]) - for _id - in resources_in_datastore]) - ) - ) - if resources.rowcount: - params = [] - for resource in resources: - new_extras = json.loads(resource[1]) - new_extras.update({'datastore_active': True}) - params.append( - {'id': resource[0], - 'extras': json.dumps(new_extras)}) - - migrate_engine.execute( - text(''' - UPDATE resource - SET extras = :extras - WHERE id = :id'''), - params) - finally: - datastore_connection.close() - datastore_engine.dispose() diff --git a/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py b/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py new file mode 100644 index 00000000000..bb1bf48258b --- /dev/null +++ b/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py @@ -0,0 +1,25 @@ +"""082 Create index creator user_id + +Revision ID: 8ea886d0ede4 +Revises: a64cf4a79182 +Create Date: 2018-09-04 18:49:17.265729 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '8ea886d0ede4' +down_revision = 'a64cf4a79182' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_index( + 'idx_package_creator_user_id', 'package', ['creator_user_id'] + ) + + +def downgrade(): + op.drop_index('idx_package_creator_user_id', 'package') diff --git a/ckan/migration/versions/082_create_index_creator_user_id.py b/ckan/migration/versions/082_create_index_creator_user_id.py deleted file mode 100644 index 2229d902cbe..00000000000 --- a/ckan/migration/versions/082_create_index_creator_user_id.py +++ /dev/null @@ -1,10 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - CREATE INDEX idx_package_creator_user_id ON package - USING BTREE (creator_user_id); - ''' - ) diff --git a/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py b/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py new file mode 100644 index 00000000000..375c1b73743 --- /dev/null +++ b/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py @@ -0,0 +1,72 @@ +"""083 Remove related items + +Revision ID: f98d8fa2a7f7 +Revises: 8ea886d0ede4 +Create Date: 2018-09-04 18:49:17.615242 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'f98d8fa2a7f7' +down_revision = '8ea886d0ede4' +branch_labels = None +depends_on = None + +WARNING = """ + +WARNING: The 'related' tables were not deleted as they currently contain data. +Once you have archived the existing data or migrated the data to +ckanext-showcase, you can safely delete the 'related' and 'related_dataset' +tables using: + + psql ckan_default -c 'BEGIN; DROP TABLE related_dataset; \\ + DROP TABLE related; COMMIT;' + +""" + + +def upgrade(): + conn = op.get_bind() + existing = conn.execute("SELECT COUNT(*) FROM related;").fetchone() + if existing[0] > 0: + print(WARNING) + return + op.drop_table('related_dataset') + op.drop_table('related') + + +def downgrade(): + + op.create_table( + 'related', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('type', sa.UnicodeText, nullable=False), + sa.Column('title', sa.UnicodeText), + sa.Column('description', sa.UnicodeText), + sa.Column('image_url', + sa.UnicodeText), sa.Column('url', sa.UnicodeText), + sa.Column('created', sa.TIMESTAMP), + sa.Column('owner_id', sa.UnicodeText), + sa.Column( + 'view_count', sa.Integer, nullable=False, server_default='0' + ), + sa.Column('featured', sa.Integer, nullable=False, server_default='0') + ) + op.create_table( + 'related_dataset', sa.Column('id', sa.UnicodeText, nullable=False), + sa.Column('dataset_id', sa.UnicodeText, nullable=False), + sa.Column('related_id', sa.UnicodeText, nullable=False), + sa.Column('status', sa.UnicodeText) + ) + op.create_primary_key('related_pkey', 'related', ['id']) + op.create_primary_key('related_dataset_pkey', 'related_dataset', ['id']) + + op.create_foreign_key( + 'related_dataset_dataset_id_fkey', 'related_dataset', 'package', + ['dataset_id'], ['id'] + ) + op.create_foreign_key( + 'related_dataset_related_id_fkey', 'related_dataset', 'related', + ['related_id'], ['id'] + ) diff --git a/ckan/migration/versions/083_remove_related_items.py b/ckan/migration/versions/083_remove_related_items.py deleted file mode 100644 index 0d1da775433..00000000000 --- a/ckan/migration/versions/083_remove_related_items.py +++ /dev/null @@ -1,28 +0,0 @@ -# encoding: utf-8 - -WARNING = """ - -WARNING: The 'related' tables were not deleted as they currently contain data. -Once you have archived the existing data or migrated the data to -ckanext-showcase, you can safely delete the 'related' and 'related_dataset' -tables using: - - psql ckan_default -c 'BEGIN; DROP TABLE related_dataset; \\ - DROP TABLE related; COMMIT;' - -""" - - -def upgrade(migrate_engine): - existing = migrate_engine.execute("SELECT COUNT(*) FROM related;")\ - .fetchone() - if existing[0] > 0: - print(WARNING) - return - - migrate_engine.execute(''' -BEGIN; -DROP TABLE related_dataset; -DROP TABLE related; -COMMIT; - ''') diff --git a/ckan/migration/versions/084_add_metadata_created.py b/ckan/migration/versions/084_add_metadata_created.py deleted file mode 100644 index 57347c54749..00000000000 --- a/ckan/migration/versions/084_add_metadata_created.py +++ /dev/null @@ -1,17 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute(''' - ALTER TABLE package_revision - ADD COLUMN metadata_created timestamp without time zone; - ALTER TABLE package - ADD COLUMN metadata_created timestamp without time zone; - - UPDATE package SET metadata_created= - (SELECT revision_timestamp - FROM package_revision - WHERE id=package.id - ORDER BY revision_timestamp ASC - LIMIT 1); - ''') diff --git a/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py b/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py new file mode 100644 index 00000000000..35a17a6a223 --- /dev/null +++ b/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py @@ -0,0 +1,38 @@ +"""084 Add metadata created + +Revision ID: d85ce5783688 +Revises: f98d8fa2a7f7 +Create Date: 2018-09-04 18:49:17.957865 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'd85ce5783688' +down_revision = 'f98d8fa2a7f7' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + 'package_revision', sa.Column('metadata_created', sa.TIMESTAMP) + ) + op.add_column('package', sa.Column('metadata_created', sa.TIMESTAMP)) + conn = op.get_bind() + conn.execute( + ''' + UPDATE package SET metadata_created= + (SELECT revision_timestamp + FROM package_revision + WHERE id=package.id + ORDER BY revision_timestamp ASC + LIMIT 1); + ''' + ) + + +def downgrade(): + op.drop_column('package', 'metadata_created') + op.drop_column('package_revision', 'metadata_created') diff --git a/ckan/migration/versions/085_adjust_activity_timestamps.py b/ckan/migration/versions/085_adjust_activity_timestamps.py deleted file mode 100644 index ddcea2ece31..00000000000 --- a/ckan/migration/versions/085_adjust_activity_timestamps.py +++ /dev/null @@ -1,23 +0,0 @@ -# encoding: utf-8 - -import datetime - - -def upgrade(migrate_engine): - u""" - The script assumes that the current timestamp was - recorded with the server's current set timezone - """ - # choose a fixed date (within DST) so migration depends only on - # server time zone not the current daylight savings state - magic_timestamp = datetime.datetime(2016, 6, 20).toordinal() - - utc_date = datetime.datetime.utcfromtimestamp(magic_timestamp) - local_date = datetime.datetime.fromtimestamp(magic_timestamp) - - if utc_date == local_date: - return - - with migrate_engine.begin() as connection: - sql = u"update activity set timestamp = timestamp + (%s - %s);" - connection.execute(sql, utc_date, local_date) diff --git a/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py b/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py new file mode 100644 index 00000000000..c6ba60df91d --- /dev/null +++ b/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py @@ -0,0 +1,44 @@ +"""085 Adjust activity timestamps + +Revision ID: f9bf3d5c4b4d +Revises: d85ce5783688 +Create Date: 2018-09-04 18:49:18.307804 + +""" +import datetime +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = 'f9bf3d5c4b4d' +down_revision = 'd85ce5783688' +branch_labels = None +depends_on = None + + +def upgrade(): + magic_timestamp = datetime.datetime(2016, 6, 20).toordinal() + + utc_date = datetime.datetime.utcfromtimestamp(magic_timestamp) + local_date = datetime.datetime.fromtimestamp(magic_timestamp) + + if utc_date == local_date: + return + + connection = op.get_bind() + sql = u"update activity set timestamp = timestamp + (%s - %s);" + connection.execute(sql, utc_date, local_date) + + +def downgrade(): + magic_timestamp = datetime.datetime(2016, 6, 20).toordinal() + + utc_date = datetime.datetime.utcfromtimestamp(magic_timestamp) + local_date = datetime.datetime.fromtimestamp(magic_timestamp) + + if utc_date == local_date: + return + + connection = op.get_bind() + sql = u"update activity set timestamp = timestamp - (%s - %s);" + connection.execute(sql, utc_date, local_date) diff --git a/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py b/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py new file mode 100644 index 00000000000..00a110a9a17 --- /dev/null +++ b/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py @@ -0,0 +1,24 @@ +"""086 Drop openid column + +Revision ID: 19663581b3bb +Revises: f9bf3d5c4b4d +Create Date: 2018-09-04 18:49:18.650337 + +""" +from alembic import op +import sqlalchemy as sa + +# revision identifiers, used by Alembic. +revision = '19663581b3bb' +down_revision = 'f9bf3d5c4b4d' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column('user', 'openid') + + +def downgrade(): + op.add_column('user', sa.Column('openid', sa.UnicodeText)) + op.create_index('idx_openid', 'user', ['openid']) diff --git a/ckan/migration/versions/086_drop_openid_column.py b/ckan/migration/versions/086_drop_openid_column.py deleted file mode 100644 index 39c71b928b8..00000000000 --- a/ckan/migration/versions/086_drop_openid_column.py +++ /dev/null @@ -1,12 +0,0 @@ -# encoding: utf-8 - -from sqlalchemy import MetaData - - -def upgrade(migrate_engine): - metadata = MetaData() - metadata.bind = migrate_engine - migrate_engine.execute(''' -ALTER TABLE "user" - DROP COLUMN openid; - ''') diff --git a/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py b/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py new file mode 100644 index 00000000000..129cd84b225 --- /dev/null +++ b/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py @@ -0,0 +1,47 @@ +"""087 Remove old authorization tables + +Revision ID: ff1b303cab77 +Revises: 19663581b3bb +Create Date: 2018-09-04 18:49:18.998454 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'ff1b303cab77' +down_revision = '19663581b3bb' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table('authorization_group_user') + op.drop_table('authorization_group') + +def downgrade(): + op.create_table( + 'authorization_group', + sa.Column('id', sa.UnicodeText, primary_key=True), + sa.Column('name', sa.UnicodeText), + sa.Column( + 'created', sa.DateTime, server_default=sa.func.current_timestamp() + ), + ) + op.create_table( + 'authorization_group_user', + sa.Column( + 'authorization_group_id', + sa.UnicodeText, + sa.ForeignKey('authorization_group.id'), + nullable=False + ), + sa.Column( + 'user_id', + sa.UnicodeText, + sa.ForeignKey('user.id'), + nullable=False + ), + sa.Column('id', sa.UnicodeText, primary_key=True) + ) diff --git a/ckan/migration/versions/087_remove_old_authorization_tables.py b/ckan/migration/versions/087_remove_old_authorization_tables.py deleted file mode 100644 index 5dbdb992be4..00000000000 --- a/ckan/migration/versions/087_remove_old_authorization_tables.py +++ /dev/null @@ -1,10 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' - DROP TABLE IF EXISTS "authorization_group_user"; - DROP TABLE IF EXISTS "authorization_group"; - ''' - ) diff --git a/ckan/migration/versions/__init__.py b/ckan/migration/versions/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/requirements.txt b/requirements.txt index beeb369c2e3..d99506fda8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ # # pip-compile --output-file requirements.txt requirements.in # +alembic==1.0.0 argparse==1.4.0 # via ofs babel==2.3.4 beaker==1.10.0 # via pylons From c77d9d8f499240e0b180b4c4bb9ed082c1467965 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Tue, 18 Sep 2018 18:28:15 +0300 Subject: [PATCH 2/6] Update CLI --- ckan/config/middleware/pylons_app.py | 2 +- ckan/controllers/api.py | 2 +- ckan/controllers/group.py | 2 +- ckan/controllers/package.py | 2 +- ckan/lib/cli.py | 10 +- ckan/migration/__init__.py | 8 + ckan/migration/env.py | 38 ++-- ...001_103676e0a497_create_existing_tables.py | 5 +- ..._86fdd8c54775_add_author_and_maintainer.py | 6 +- .../003_f22b4f5241a5_add_user_object.py | 6 +- .../004_f92ee205e46d_add_group_object.py | 7 +- ...5_12c2232c15f5_add_authorization_tables.py | 6 +- .../versions/006_c83955e7acb6_add_ratings.py | 6 +- .../007_1928d4af1cda_add_system_roles.py | 5 +- .../008_e8283ffb257e_update_vdm_ids.py | 9 +- ...09_b739a48de5c4_add_creation_timestamps.py | 5 +- .../010_a6f13bf14d0c_add_user_about.py | 5 +- ..._866f6370b4ac_add_package_search_vector.py | 5 +- .../012_e5ca33a5d445_add_resources.py | 5 +- .../versions/013_8a3a5af39797_add_hash.py | 5 +- .../versions/014_93519b684820_hash_2.py | 5 +- .../015_6d8ffebcaf54_remove_state_object.py | 5 +- .../016_37ada738328e_uuids_everywhere.py | 5 +- .../017_1250b2ff3e36_add_pkg_relationships.py | 5 +- .../018_05a0778051ca_adjust_licenses.py | 8 +- ...19_b2eb6f34a638_pkg_relationships_state.py | 5 +- .../020_69a0b0efc609_add_changeset.py | 5 +- ...021_765143af2ba3_postgresql_upgrade_sql.py | 5 +- .../022_7b324ca6c0dc_add_group_extras.py | 5 +- .../023_87fdd05f0744_add_harvesting.py | 5 +- ...024_12981fe12484_add_harvested_document.py | 5 +- ...5_b581622ad327_add_authorization_groups.py | 5 +- ...615b25af443_authorization_group_user_pk.py | 6 +- .../027_11e5745c6fc9_adjust_harvester.py | 5 +- ...cdd68fe9ba21_drop_harvest_source_status.py | 6 +- .../029_1bfdf4240915_version_groups.py | 5 +- ...b16cbf164c8a_additional_user_attributes.py | 5 +- ...1_1b05245167d6_move_openid_to_new_field.py | 6 +- ...1422d_add_extra_info_field_to_resources.py | 5 +- ...df15_auth_group_user_id_add_conditional.py | 5 +- .../034_6c600693af5b_resource_group_table.py | 11 +- ..._81148ccebd6c_harvesting_doc_versioning.py | 5 +- .../036_ecaa8b38782f_lockdown_roles.py | 6 +- .../037_edcf3b8c3c1b_role_anon_editor.py | 6 +- ...38_fd6622e3d964_delete_migration_tables.py | 8 +- ...9_cca459c76d45_add_expired_id_and_dates.py | 5 +- .../040_500a08f4818e_reset_key_on_user.py | 5 +- .../041_6817d4e3bdc3_resource_new_fields.py | 5 +- .../042_da65e2877034_user_revision_indexes.py | 9 +- .../043_bd38cd6502b2_drop_postgres_search.py | 5 +- .../044_4190eeeb8d73_add_task_status.py | 5 +- .../045_54e3f155d945_user_name_unique.py | 6 +- .../046_b69e9b80396f_drop_changesets.py | 5 +- ...83a7c406926_rename_package_group_member.py | 5 +- ...a7011172b3f_add_activity_streams_tables.py | 5 +- ..._e0c06c2177b5_add_group_approval_status.py | 5 +- ...050_01a6b058cb7f_term_translation_table.py | 5 +- .../051_a4fb0d85ced6_add_tag_vocabulary.py | 5 +- ...2_ba693d64c6d7_update_member_capacities.py | 6 +- .../053_9d051a099097_add_group_logo.py | 5 +- ..._da21b38da4db_add_resource_created_date.py | 6 +- ...db947bf_update_user_and_activity_detail.py | 6 +- .../056_11af3215ae89_add_related_table.py | 5 +- .../versions/057_660a5aae527e_tracking.py | 5 +- .../058_bd36d1826a5d_add_follower_tables.py | 5 +- ...9291bb46f352_add_related_count_and_flag.py | 5 +- .../060_31ad11c518fc_add_system_info_table.py | 5 +- ...1_338d460bc460_add_follower_group_table.py | 11 +- .../062_6deb2bbab394_add_dashboard_table.py | 5 +- .../versions/063_8b633852cb7a_org_changes.py | 5 +- ...4f8becd4919a_add_email_last_sent_column.py | 5 +- ...439e_add_email_notifications_preference.py | 5 +- .../066_ad16b3bd8cb6_default_package_type.py | 6 +- ...067_266c110eafec_turn_extras_to_strings.py | 6 +- ...8_e33a5f2b2a84_add_package_extras_index.py | 6 +- ...5cdb_resource_url_and_metadata_modified.py | 5 +- ...12fa7_add_activity_and_resource_indexes.py | 5 +- ...081ef73a_add_state_column_to_user_table.py | 5 +- .../072_08dcb9233ad7_add_resource_view.py | 5 +- ...08be3_update_resource_view_resource_id_.py | 6 +- ...074_a4ca55f0f45e_remove_resource_groups.py | 5 +- .../075_9cdc88c8896a_rename_view_plugins.py | 6 +- .../076_59995aa965c0_rename_view_plugins_2.py | 6 +- ...171a04d86d_add_revisions_to_system_info.py | 5 +- ...078_ae821876532a_remove_old_authz_model.py | 6 +- ...79_e0177a15d2c9_resource_revision_index.py | 6 +- .../080_8224d872c64f_continuity_id_indexes.py | 6 +- .../081_a64cf4a79182_set_datastore_active.py | 6 +- ...a886d0ede4_create_index_creator_user_id.py | 6 +- .../083_f98d8fa2a7f7_remove_related_items.py | 5 +- .../084_d85ce5783688_add_metadata_created.py | 5 +- ...f9bf3d5c4b4d_adjust_activity_timestamps.py | 5 +- .../086_19663581b3bb_drop_openid_column.py | 5 +- ...03cab77_remove_old_authorization_tables.py | 9 +- ...d397f70903_remove_migrate_version_table.py | 23 +++ ckan/model/__init__.py | 109 +++++++---- ckan/tests/legacy/test_coding_standards.py | 177 +++--------------- ckan/tests/test_coding_standards.py | 171 +++++++++-------- 98 files changed, 617 insertions(+), 418 deletions(-) create mode 100644 ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index 73b714509a7..bac6276972d 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -267,7 +267,7 @@ def execute_on_completion(application, config, callback): def inner(environ, start_response): try: result = application(environ, start_response) - except: + except Exception: callback(environ) raise # paste.fileapp converts non-file responses into list diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 0093aed72e8..d73696174ff 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -415,7 +415,7 @@ def group_autocomplete(self): limit = request.params.get('limit', 20) try: limit = int(limit) - except: + except Exception: limit = 20 limit = min(50, limit) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 3bbe20fbad5..680329f6154 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -802,7 +802,7 @@ def history(self, id): revision_dict['timestamp']) try: dayHorizon = int(request.params.get('days')) - except: + except Exception: dayHorizon = 30 dayAge = (datetime.datetime.now() - revision_date).days if dayAge >= dayHorizon: diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index fb28d93aac1..6a000c07725 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -472,7 +472,7 @@ def history(self, id): revision_dict['timestamp']) try: dayHorizon = int(request.params.get('days')) - except: + except Exception: dayHorizon = 30 dayAge = (datetime.datetime.now() - revision_date).days if dayAge >= dayHorizon: diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index aecc40c2f2e..f9499833eb7 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -351,7 +351,8 @@ class ManageDb(CkanCommand): def command(self): cmd = self.args[0] - self._load_config(cmd!='upgrade') + self._load_config(cmd != 'upgrade') + import ckan.model as model import ckan.lib.search as search @@ -375,10 +376,9 @@ def command(self): if self.verbose: print('Cleaning DB: SUCCESS') elif cmd == 'upgrade': - if len(self.args) > 1: - model.repo.upgrade_db(self.args[1]) - else: - model.repo.upgrade_db() + model.repo.upgrade_db(*self.args[1:]) + elif cmd == 'downgrade': + model.repo.downgrade_db(*self.args[1:]) elif cmd == 'version': self.version() elif cmd == 'create-from-model': diff --git a/ckan/migration/__init__.py b/ckan/migration/__init__.py index e69de29bb2d..912ce6b7da9 100644 --- a/ckan/migration/__init__.py +++ b/ckan/migration/__init__.py @@ -0,0 +1,8 @@ +# encoding: utf-8 + + +def skip_based_on_legacy_engine_version(op, filename): + conf = op.get_context().config + version = conf.get_main_option(u'sqlalchemy_migrate_version') + if version: + return int(version) >= int(filename.split(u'_', 1)[0]) diff --git a/ckan/migration/env.py b/ckan/migration/env.py index 1b451f47470..fe181a6908c 100644 --- a/ckan/migration/env.py +++ b/ckan/migration/env.py @@ -1,7 +1,11 @@ +# encoding: utf-8 + from __future__ import with_statement from alembic import context from sqlalchemy import engine_from_config, pool -from logging.config import fileConfig +# from logging.config import fileConfig +from ckan.model import init_model +from ckan.model.meta import metadata # this is the Alembic Config object, which provides # access to the values within the .ini file in use. @@ -9,14 +13,12 @@ # Interpret the config file for Python logging. # This line sets up loggers basically. -fileConfig(config.config_file_name) +# fileConfig(config.config_file_name) # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from ckan.model import init_model -from ckan.model.meta import metadata target_metadata = metadata # other values from the config, defined by the needs of env.py, @@ -37,9 +39,10 @@ def run_migrations_offline(): script output. """ - url = config.get_main_option("sqlalchemy.url") + url = config.get_main_option(u"sqlalchemy.url") context.configure( - url=url, target_metadata=target_metadata, literal_binds=True) + url=url, target_metadata=target_metadata, literal_binds=True + ) with context.begin_transaction(): context.run_migrations() @@ -54,18 +57,17 @@ def run_migrations_online(): """ connectable = engine_from_config( config.get_section(config.config_ini_section), - prefix='sqlalchemy.', - poolclass=pool.NullPool) - with connectable.connect() as connection: - init_model(connection) - - context.configure( - connection=connection, - target_metadata=target_metadata - ) - - with context.begin_transaction(): - context.run_migrations() + prefix=u'sqlalchemy.', + poolclass=pool.NullPool + ) + connection = connectable.connect() + init_model(connection) + + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + if context.is_offline_mode(): run_migrations_offline() diff --git a/ckan/migration/versions/001_103676e0a497_create_existing_tables.py b/ckan/migration/versions/001_103676e0a497_create_existing_tables.py index 45b5ed93ee0..4e1b2552d2d 100644 --- a/ckan/migration/versions/001_103676e0a497_create_existing_tables.py +++ b/ckan/migration/versions/001_103676e0a497_create_existing_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """Create existing tables Revision ID: 103676e0a497 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '103676e0a497' down_revision = None @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'state', diff --git a/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py b/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py index b31b01e7daa..0bc66fc52fd 100644 --- a/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py +++ b/ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """Add author and maintainer Revision ID: 86fdd8c54775 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '86fdd8c54775' down_revision = '103676e0a497' @@ -23,6 +24,9 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return + for column in _columns: op.add_column('package', sa.Column(column, sa.UnicodeText)) op.add_column('package_revision', sa.Column(column, sa.UnicodeText)) diff --git a/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py b/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py index d874d014d2c..f95670b19b9 100644 --- a/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py +++ b/ckan/migration/versions/003_f22b4f5241a5_add_user_object.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """Add user object Revision ID: f22b4f5241a5 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'f22b4f5241a5' down_revision = '86fdd8c54775' @@ -16,6 +17,9 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return + op.create_table( 'user', sa.Column('id', sa.UnicodeText, primary_key=True), sa.Column('name', sa.UnicodeText), sa.Column('apikey', sa.UnicodeText) diff --git a/ckan/migration/versions/004_f92ee205e46d_add_group_object.py b/ckan/migration/versions/004_f92ee205e46d_add_group_object.py index fb833544ef9..606edd70da5 100644 --- a/ckan/migration/versions/004_f92ee205e46d_add_group_object.py +++ b/ckan/migration/versions/004_f92ee205e46d_add_group_object.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """Add group object Revision ID: f92ee205e46d @@ -5,11 +6,9 @@ Create Date: 2018-09-04 17:22:50.675045 """ -import uuid - from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'f92ee205e46d' down_revision = 'f22b4f5241a5' @@ -18,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'group', diff --git a/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py b/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py index db146351be5..0ad3179aede 100644 --- a/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py +++ b/ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """005 Add authorization tables Revision ID: 12c2232c15f5 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '12c2232c15f5' down_revision = 'f92ee205e46d' @@ -16,6 +17,9 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return + op.create_table( 'role_action', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/006_c83955e7acb6_add_ratings.py b/ckan/migration/versions/006_c83955e7acb6_add_ratings.py index b71585cd76e..f97d13b112c 100644 --- a/ckan/migration/versions/006_c83955e7acb6_add_ratings.py +++ b/ckan/migration/versions/006_c83955e7acb6_add_ratings.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """006 Add ratings Revision ID: c83955e7acb6 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'c83955e7acb6' down_revision = '12c2232c15f5' @@ -16,6 +17,9 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return + op.create_table( 'rating', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py b/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py index dafc500053e..f59e3b85629 100644 --- a/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py +++ b/ckan/migration/versions/007_1928d4af1cda_add_system_roles.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """007 Add system roles Revision ID: 1928d4af1cda @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '1928d4af1cda' down_revision = 'c83955e7acb6' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'system_role', sa.Column( diff --git a/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py b/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py index 02e4850ce80..dcfa4d1bd8d 100644 --- a/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py +++ b/ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """008 Update vdm ids Revision ID: e8283ffb257e @@ -9,7 +10,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e8283ffb257e' down_revision = '1928d4af1cda' @@ -23,9 +24,13 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table in foreign_tables: op.drop_column(table, 'revision_id') - op.alter_column('revision', 'id', type_=sa.UnicodeText, server_default=None) + op.alter_column( + 'revision', 'id', type_=sa.UnicodeText, server_default=None + ) for table in foreign_tables: op.add_column( table, diff --git a/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py b/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py index a1574896d08..0158a482145 100644 --- a/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py +++ b/ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """009 Add creation timestamps Revision ID: b739a48de5c4 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'b739a48de5c4' down_revision = 'e8283ffb257e' @@ -18,6 +19,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table in domain_obj_names: op.add_column(table, sa.Column('created', sa.TIMESTAMP)) diff --git a/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py b/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py index d791cda98c4..589d3112fb4 100644 --- a/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py +++ b/ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """010 Add user about Revision ID: a6f13bf14d0c @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'a6f13bf14d0c' down_revision = 'b739a48de5c4' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('user', sa.Column('about', sa.Text)) diff --git a/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py b/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py index cb337f0b158..3b6f60efaff 100644 --- a/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py +++ b/ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """011 Add package search vector Revision ID: 866f6370b4ac @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '866f6370b4ac' down_revision = 'a6f13bf14d0c' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'package_search', sa.Column( diff --git a/ckan/migration/versions/012_e5ca33a5d445_add_resources.py b/ckan/migration/versions/012_e5ca33a5d445_add_resources.py index bcb2ef9c80a..e75b5150d38 100644 --- a/ckan/migration/versions/012_e5ca33a5d445_add_resources.py +++ b/ckan/migration/versions/012_e5ca33a5d445_add_resources.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """012 Add resources Revision ID: e5ca33a5d445 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e5ca33a5d445' down_revision = '866f6370b4ac' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'package_resource', sa.Column('id', sa.Integer, primary_key=True), diff --git a/ckan/migration/versions/013_8a3a5af39797_add_hash.py b/ckan/migration/versions/013_8a3a5af39797_add_hash.py index 0c288e12314..86ee631cde5 100644 --- a/ckan/migration/versions/013_8a3a5af39797_add_hash.py +++ b/ckan/migration/versions/013_8a3a5af39797_add_hash.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """013 Add hash Revision ID: 8a3a5af39797 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '8a3a5af39797' down_revision = 'e5ca33a5d445' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('package_resource', sa.Column('hash', sa.UnicodeText)) diff --git a/ckan/migration/versions/014_93519b684820_hash_2.py b/ckan/migration/versions/014_93519b684820_hash_2.py index d9f16989de1..b6c82620041 100644 --- a/ckan/migration/versions/014_93519b684820_hash_2.py +++ b/ckan/migration/versions/014_93519b684820_hash_2.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """014 Hash 2 Revision ID: 93519b684820 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '93519b684820' down_revision = '8a3a5af39797' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'package_resource_revision', sa.Column('hash', sa.UnicodeText) ) diff --git a/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py b/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py index 9d1988fefd0..8f3cadddb89 100644 --- a/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py +++ b/ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """015 Remove state_object Revision ID: 6d8ffebcaf54 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '6d8ffebcaf54' down_revision = '93519b684820' @@ -22,6 +23,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table_name in stateful_tables: op.add_column(table_name, sa.Column('state', sa.UnicodeText)) op.drop_column(table_name, 'state_id') diff --git a/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py b/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py index 75ef15f8591..7ccc609fe4b 100644 --- a/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py +++ b/ckan/migration/versions/016_37ada738328e_uuids_everywhere.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """016 Uuids everywhere Revision ID: 37ada738328e @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '37ada738328e' down_revision = '6d8ffebcaf54' @@ -50,6 +51,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table, column in foreign_keys: op.drop_column(table, column) for table in continuity: diff --git a/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py b/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py index 767f9feb7ac..9d606d7a6dc 100644 --- a/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py +++ b/ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """017 Add pkg_relationships Revision ID: 1250b2ff3e36 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '1250b2ff3e36' down_revision = '37ada738328e' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'package_relationship', diff --git a/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py b/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py index b8c72272284..5f52be55a30 100644 --- a/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py +++ b/ckan/migration/versions/018_05a0778051ca_adjust_licenses.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """018 Adjust licenses Revision ID: 05a0778051ca @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '05a0778051ca' down_revision = '1250b2ff3e36' @@ -18,6 +19,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table in tables: op.drop_column(table, 'license_id') op.add_column(table, sa.Column('license_id', sa.UnicodeText)) @@ -29,8 +32,7 @@ def downgrade(): op.create_table( 'license', sa.Column('id', sa.Integer, primary_key=True, nullable=False), - sa.Column('name', sa.Unicode(100)), - sa.Column('state', sa.UnicodeText) + sa.Column('name', sa.Unicode(100)), sa.Column('state', sa.UnicodeText) ) for table in tables: op.drop_column(table, 'license_id') diff --git a/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py b/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py index 9fd7f1dba87..ad23e8c7e24 100644 --- a/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py +++ b/ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """019 Pkg relationships_state Revision ID: b2eb6f34a638 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'b2eb6f34a638' down_revision = '05a0778051ca' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('package_relationship', sa.Column('state', sa.UnicodeText)) op.add_column( 'package_relationship_revision', sa.Column('state', sa.UnicodeText) diff --git a/ckan/migration/versions/020_69a0b0efc609_add_changeset.py b/ckan/migration/versions/020_69a0b0efc609_add_changeset.py index e8335313a23..e8fc4db68f2 100644 --- a/ckan/migration/versions/020_69a0b0efc609_add_changeset.py +++ b/ckan/migration/versions/020_69a0b0efc609_add_changeset.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """020 Add changeset Revision ID: 69a0b0efc609 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '69a0b0efc609' down_revision = 'b2eb6f34a638' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'changeset', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py b/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py index e5b468af436..babdd7c8199 100644 --- a/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py +++ b/ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """021 Postgresql upgrade.sql Revision ID: 765143af2ba3 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '765143af2ba3' down_revision = '69a0b0efc609' @@ -93,6 +94,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for name, table, columns in indexes: op.create_index(name, table, columns) diff --git a/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py b/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py index d7230ca9000..ec46bf5a4ce 100644 --- a/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py +++ b/ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """022 Add group_extras Revision ID: 7b324ca6c0dc @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '7b324ca6c0dc' down_revision = '765143af2ba3' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'group_extra', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py b/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py index e7cb28f1afe..6b5889a70a3 100644 --- a/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py +++ b/ckan/migration/versions/023_87fdd05f0744_add_harvesting.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """023 Add harvesting Revision ID: 87fdd05f0744 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '87fdd05f0744' down_revision = '7b324ca6c0dc' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'harvest_source', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/024_12981fe12484_add_harvested_document.py b/ckan/migration/versions/024_12981fe12484_add_harvested_document.py index a4ecc117e9f..7ea7a11c578 100644 --- a/ckan/migration/versions/024_12981fe12484_add_harvested_document.py +++ b/ckan/migration/versions/024_12981fe12484_add_harvested_document.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """024 Add harvested_document Revision ID: 12981fe12484 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '12981fe12484' down_revision = '87fdd05f0744' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'harvested_document', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py b/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py index 0c78a31c4f7..fe4b1e9eed5 100644 --- a/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py +++ b/ckan/migration/versions/025_b581622ad327_add_authorization_groups.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """025 Add authorization groups Revision ID: b581622ad327 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'b581622ad327' down_revision = '12981fe12484' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'authorization_group', sa.Column('id', sa.UnicodeText, primary_key=True), diff --git a/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py b/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py index a15ceecfcdd..7af7c45da37 100644 --- a/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py +++ b/ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """026 Authorization group user pk Revision ID: 3615b25af443 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '3615b25af443' down_revision = 'b581622ad327' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py b/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py index 8c445bc851d..c1b76833eba 100644 --- a/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py +++ b/ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """027 Adjust harvester Revision ID: 11e5745c6fc9 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '11e5745c6fc9' down_revision = '3615b25af443' @@ -18,6 +19,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column(table, sa.Column('guid', sa.UnicodeText, server_default=u'')) op.add_column( table, diff --git a/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py b/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py index f41b7d2af9c..e11eea43d17 100644 --- a/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py +++ b/ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """028 Drop harvest_source_status Revision ID: cdd68fe9ba21 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'cdd68fe9ba21' down_revision = '11e5745c6fc9' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.alter_column('harvest_source', 'status', nullable=False) diff --git a/ckan/migration/versions/029_1bfdf4240915_version_groups.py b/ckan/migration/versions/029_1bfdf4240915_version_groups.py index e7a72103aaa..b62ec6ddcf7 100644 --- a/ckan/migration/versions/029_1bfdf4240915_version_groups.py +++ b/ckan/migration/versions/029_1bfdf4240915_version_groups.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """029 Version groups Revision ID: 1bfdf4240915 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '1bfdf4240915' down_revision = 'cdd68fe9ba21' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'group_revision', sa.Column('id', sa.UnicodeText, primary_key=True), sa.Column('name', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py b/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py index d918934c70d..8999eb2d509 100644 --- a/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py +++ b/ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """030 Additional user_attributes Revision ID: b16cbf164c8a @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'b16cbf164c8a' down_revision = '1bfdf4240915' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('user', sa.Column('openid', sa.UnicodeText)) op.add_column('user', sa.Column('password', sa.UnicodeText)) op.add_column('user', sa.Column('fullname', sa.UnicodeText)) diff --git a/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py b/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py index ad5d579dafa..d301563c6fc 100644 --- a/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py +++ b/ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """031 Move openid to new_field Revision ID: 1b05245167d6 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '1b05245167d6' down_revision = 'b16cbf164c8a' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py b/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py index 7ad8607049e..0daae5d617d 100644 --- a/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py +++ b/ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """032 Add extra info field_to_resources Revision ID: d89e0731422d @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'd89e0731422d' down_revision = '1b05245167d6' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('package_resource', sa.Column('extras', sa.UnicodeText)) op.add_column( 'package_resource_revision', sa.Column('extras', sa.UnicodeText) diff --git a/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py b/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py index a2feaa1037f..a9e53fdc45a 100644 --- a/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py +++ b/ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """033 Auth group user id_add_conditional Revision ID: 6da92ef2df15 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '6da92ef2df15' down_revision = 'd89e0731422d' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('authorization_group_user', sa.Column('id', sa.UnicodeText)) op.create_primary_key( 'authorization_group_user_pkey', 'authorization_group_user', ['id'] diff --git a/ckan/migration/versions/034_6c600693af5b_resource_group_table.py b/ckan/migration/versions/034_6c600693af5b_resource_group_table.py index 8696b41193d..c79a1240aa2 100644 --- a/ckan/migration/versions/034_6c600693af5b_resource_group_table.py +++ b/ckan/migration/versions/034_6c600693af5b_resource_group_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """034 Resource group table Revision ID: 6c600693af5b @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '6c600693af5b' down_revision = '6da92ef2df15' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'resource_group', sa.Column('id', sa.UnicodeText, primary_key=True), @@ -59,7 +62,8 @@ def upgrade(): op.rename_table('package_resource_revision', 'resource_revision') op.execute('ALTER INDEX package_resource_pkey RENAME TO resource_pkey') op.execute( - 'ALTER INDEX package_resource_revision_pkey RENAME TO resource_revision_pkey' + 'ALTER INDEX package_resource_revision_pkey ' + 'RENAME TO resource_revision_pkey' ) op.drop_constraint( @@ -113,7 +117,8 @@ def downgrade(): op.rename_table('resource', 'package_resource') op.execute('ALTER INDEX resource_pkey RENAME TO package_resource_pkey') op.execute( - 'ALTER INDEX resource_revision_pkey RENAME TO package_resource_revision_pkey' + 'ALTER INDEX resource_revision_pkey ' + 'RENAME TO package_resource_revision_pkey' ) op.alter_column( diff --git a/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py b/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py index 71fddce62d0..a2b76d0348f 100644 --- a/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py +++ b/ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """035 Harvesting doc versioning Revision ID: 81148ccebd6c @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '81148ccebd6c' down_revision = '6c600693af5b' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'harvested_document_revision', sa.Column('id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py b/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py index c37f1ba7957..9210267f4e8 100644 --- a/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py +++ b/ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """036 Lockdown roles Revision ID: ecaa8b38782f @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'ecaa8b38782f' down_revision = '81148ccebd6c' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py b/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py index 09f9e3cb7e1..6be9677d1fe 100644 --- a/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py +++ b/ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """037 Role anon_editor Revision ID: edcf3b8c3c1b @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'edcf3b8c3c1b' down_revision = 'ecaa8b38782f' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py b/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py index 6ef7589caf4..8127d17b74d 100644 --- a/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py +++ b/ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """038 Delete migration tables Revision ID: fd6622e3d964 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'fd6622e3d964' down_revision = 'edcf3b8c3c1b' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_table('harvested_document_revision') op.drop_table('harvested_document') op.drop_table('harvesting_job') @@ -59,7 +62,8 @@ def downgrade(): sa.Column('package_id', sa.UnicodeText), sa.Column('state', sa.UnicodeText), sa.Column('revision_id', sa.UnicodeText, nullable=False), - sa.Column('guid', sa.UnicodeText), sa.Column('created', sa.TIMESTAMP), + sa.Column('guid', sa.UnicodeText), + sa.Column('created', sa.TIMESTAMP), ) op.create_table( diff --git a/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py b/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py index 0ddaf80778c..a2594f40aa1 100644 --- a/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py +++ b/ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """039 Add expired id and_dates Revision ID: cca459c76d45 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'cca459c76d45' down_revision = 'fd6622e3d964' @@ -130,6 +131,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table in tables: op.add_column(table, sa.Column('expired_id', sa.UnicodeText)) diff --git a/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py b/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py index 7cc2c0ef66e..e46191498fa 100644 --- a/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py +++ b/ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """040 Reset key on user Revision ID: 500a08f4818e @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '500a08f4818e' down_revision = 'cca459c76d45' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('user', sa.Column('reset_key', sa.UnicodeText)) diff --git a/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py b/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py index 1fcdd72f884..699a0e1dc57 100644 --- a/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py +++ b/ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """041 Resource new fields Revision ID: 6817d4e3bdc3 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '6817d4e3bdc3' down_revision = '500a08f4818e' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return for table in ('resource', 'resource_revision'): op.add_column(table, sa.Column('name', sa.UnicodeText)) op.add_column(table, sa.Column('resource_type', sa.UnicodeText)) diff --git a/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py b/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py index 26e10d612b1..6870ee9b80a 100644 --- a/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py +++ b/ckan/migration/versions/042_da65e2877034_user_revision_indexes.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """042 User revision indexes Revision ID: da65e2877034 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'da65e2877034' down_revision = '6817d4e3bdc3' @@ -16,12 +17,16 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index('idx_revision_author', 'revision', ['author']) op.create_index('idx_openid', 'user', ['openid']) op.create_index( 'idx_user_name_index', 'user', [ sa.text( - '''(CASE WHEN ("user".fullname IS NULL OR "user".fullname = '') THEN "user".name ELSE "user".fullname END)''' + '(CASE WHEN ' + '("user".fullname IS NULL OR "user".fullname = \'\') ' + 'THEN "user".name ELSE "user".fullname END)' ) ] ) diff --git a/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py b/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py index fb7411a8cda..252b47d522b 100644 --- a/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py +++ b/ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """043 Drop postgres search Revision ID: bd38cd6502b2 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'bd38cd6502b2' down_revision = 'da65e2877034' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_table('package_search') diff --git a/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py b/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py index eb3367b4ef7..9eafced4d73 100644 --- a/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py +++ b/ckan/migration/versions/044_4190eeeb8d73_add_task_status.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """044 Add task status Revision ID: 4190eeeb8d73 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '4190eeeb8d73' down_revision = 'bd38cd6502b2' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'task_status', sa.Column('id', sa.UnicodeText, nullable=False), sa.Column('entity_id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/045_54e3f155d945_user_name_unique.py b/ckan/migration/versions/045_54e3f155d945_user_name_unique.py index 42a7b006538..cdaf3671edf 100644 --- a/ckan/migration/versions/045_54e3f155d945_user_name_unique.py +++ b/ckan/migration/versions/045_54e3f155d945_user_name_unique.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """045 User name unique Revision ID: 54e3f155d945 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '54e3f155d945' down_revision = '4190eeeb8d73' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_unique_constraint('user_name_key', 'user', ['name']) diff --git a/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py b/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py index 062d2a2d9af..aa71661240a 100644 --- a/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py +++ b/ckan/migration/versions/046_b69e9b80396f_drop_changesets.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """046 Drop changesets Revision ID: b69e9b80396f @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'b69e9b80396f' down_revision = '54e3f155d945' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_table('change') op.drop_table('changemask') op.drop_table('changeset') diff --git a/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py b/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py index 7467535727a..28a74e478a1 100644 --- a/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py +++ b/ckan/migration/versions/047_883a7c406926_rename_package_group_member.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """047 Rename package_group_member Revision ID: 883a7c406926 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '883a7c406926' down_revision = 'b69e9b80396f' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.rename_table('package_group', 'member') op.rename_table('package_group_revision', 'member_revision') diff --git a/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py b/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py index 95c201da4be..f618addd1b9 100644 --- a/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py +++ b/ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """048 Add activity streams tables Revision ID: 4a7011172b3f @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '4a7011172b3f' down_revision = '883a7c406926' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'activity', sa.Column('id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py b/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py index 10717202768..0198878dbd9 100644 --- a/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py +++ b/ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """049 Add group approval status Revision ID: e0c06c2177b5 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e0c06c2177b5' down_revision = '4a7011172b3f' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('group', sa.Column('approval_status', sa.UnicodeText)) op.add_column( 'group_revision', sa.Column('approval_status', sa.UnicodeText) diff --git a/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py b/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py index 9894644156c..9f070c90d02 100644 --- a/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py +++ b/ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """050 Term translation table Revision ID: 01a6b058cb7f @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '01a6b058cb7f' down_revision = 'e0c06c2177b5' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'term_translation', sa.Column('term', sa.UnicodeText, nullable=False), sa.Column('term_translation', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py b/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py index 4557f84a4a2..c99ad4a4b75 100644 --- a/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py +++ b/ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """051 Add tag vocabulary Revision ID: a4fb0d85ced6 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'a4fb0d85ced6' down_revision = '01a6b058cb7f' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_constraint('tag_name_key', 'tag') op.create_table( 'vocabulary', sa.Column('id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py b/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py index 0a66b12607b..a5fd64565c2 100644 --- a/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py +++ b/ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """052 Update member capacities Revision ID: ba693d64c6d7 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'ba693d64c6d7' down_revision = 'a4fb0d85ced6' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/053_9d051a099097_add_group_logo.py b/ckan/migration/versions/053_9d051a099097_add_group_logo.py index e0219cfc481..ba83d1af583 100644 --- a/ckan/migration/versions/053_9d051a099097_add_group_logo.py +++ b/ckan/migration/versions/053_9d051a099097_add_group_logo.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """053 Add group logo Revision ID: 9d051a099097 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '9d051a099097' down_revision = 'ba693d64c6d7' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('group', sa.Column('image_url', sa.UnicodeText)) op.add_column('group_revision', sa.Column('image_url', sa.UnicodeText)) diff --git a/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py b/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py index 0c33c3608eb..50ceb698617 100644 --- a/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py +++ b/ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """054 Add resource created date Revision ID: da21b38da4db @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'da21b38da4db' down_revision = '9d051a099097' @@ -16,6 +17,9 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return + op.add_column('resource', sa.Column('created', sa.TIMESTAMP)) op.add_column('resource_revision', sa.Column('created', sa.TIMESTAMP)) diff --git a/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py b/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py index e8d83abeafd..92ed5850245 100644 --- a/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py +++ b/ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """055 Update user and activity_detail Revision ID: 048f7db947bf @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '048f7db947bf' down_revision = 'da21b38da4db' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.alter_column('activity_detail', 'activity_id', nullable=True) op.alter_column('user', 'name', nullable=False) diff --git a/ckan/migration/versions/056_11af3215ae89_add_related_table.py b/ckan/migration/versions/056_11af3215ae89_add_related_table.py index ce048574c9d..b9f92ffab60 100644 --- a/ckan/migration/versions/056_11af3215ae89_add_related_table.py +++ b/ckan/migration/versions/056_11af3215ae89_add_related_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """056 Add related table Revision ID: 11af3215ae89 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '11af3215ae89' down_revision = '048f7db947bf' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'related', sa.Column('id', sa.UnicodeText, nullable=False), sa.Column('type', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/057_660a5aae527e_tracking.py b/ckan/migration/versions/057_660a5aae527e_tracking.py index 9cadfe72fcc..c3c7a09cd2c 100644 --- a/ckan/migration/versions/057_660a5aae527e_tracking.py +++ b/ckan/migration/versions/057_660a5aae527e_tracking.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """057 Tracking Revision ID: 660a5aae527e @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '660a5aae527e' down_revision = '11af3215ae89' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'tracking_raw', sa.Column('user_key', sa.String(100), nullable=False), sa.Column('url', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py b/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py index a1f5a28742e..78cef1d6fec 100644 --- a/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py +++ b/ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """058 Add follower tables Revision ID: bd36d1826a5d @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'bd36d1826a5d' down_revision = '660a5aae527e' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'user_following_dataset', sa.Column('follower_id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py b/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py index 8371a37a0c7..b75fb1b6061 100644 --- a/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py +++ b/ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """059 Add related count and_flag Revision ID: 9291bb46f352 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '9291bb46f352' down_revision = 'bd36d1826a5d' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'related', sa.Column( diff --git a/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py b/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py index 5639f9302da..ba3d7a57217 100644 --- a/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py +++ b/ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """060 Add system info table Revision ID: 31ad11c518fc @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '31ad11c518fc' down_revision = '9291bb46f352' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'system_info', diff --git a/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py b/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py index a3a9185fa5a..624d17243b6 100644 --- a/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py +++ b/ckan/migration/versions/061_338d460bc460_add_follower_group_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """061 Add follower group_table Revision ID: 338d460bc460 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '338d460bc460' down_revision = '31ad11c518fc' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'user_following_group', sa.Column('follower_id', sa.UnicodeText, nullable=False), @@ -30,16 +33,14 @@ def upgrade(): op.create_foreign_key( 'user_following_group_user_id_fkey', 'user_following_group', - 'user', ['follower_id'], - ['id'], + 'user', ['follower_id'], ['id'], onupdate='CASCADE', ondelete='CASCADE' ) op.create_foreign_key( 'user_following_group_group_id_fkey', 'user_following_group', - 'group', ['object_id'], - ['id'], + 'group', ['object_id'], ['id'], onupdate='CASCADE', ondelete='CASCADE' ) diff --git a/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py b/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py index 8d3e063b17c..86b77f20400 100644 --- a/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py +++ b/ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """062 Add dashboard table Revision ID: 6deb2bbab394 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '6deb2bbab394' down_revision = '338d460bc460' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'dashboard', sa.Column('user_id', sa.UnicodeText, nullable=False), sa.Column('activity_stream_last_viewed', sa.TIMESTAMP, nullable=False) diff --git a/ckan/migration/versions/063_8b633852cb7a_org_changes.py b/ckan/migration/versions/063_8b633852cb7a_org_changes.py index 3569ccbc35b..b02e08505e6 100644 --- a/ckan/migration/versions/063_8b633852cb7a_org_changes.py +++ b/ckan/migration/versions/063_8b633852cb7a_org_changes.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """063 Org changes Revision ID: 8b633852cb7a @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '8b633852cb7a' down_revision = '6deb2bbab394' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'user', sa.Column('sysadmin', sa.Boolean, server_default='FALSE') diff --git a/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py b/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py index 96c571df8f4..6f8f6c89fbe 100644 --- a/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py +++ b/ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """064 Add email_last_sent_column Revision ID: 4f8becd4919a @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '4f8becd4919a' down_revision = '8b633852cb7a' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'dashboard', sa.Column( diff --git a/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py b/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py index aa594e63be8..1fb44518cb8 100644 --- a/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py +++ b/ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """065 Add email notifications preference Revision ID: 1fab0bc6439e @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '1fab0bc6439e' down_revision = '4f8becd4919a' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'user', sa.Column( diff --git a/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py b/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py index dd56a043af1..f6e319074d6 100644 --- a/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py +++ b/ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """066 Default package type Revision ID: ad16b3bd8cb6 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'ad16b3bd8cb6' down_revision = '1fab0bc6439e' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py b/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py index edbc2cdc496..153ab6fed78 100644 --- a/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py +++ b/ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """067 Turn extras to strings Revision ID: 266c110eafec @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '266c110eafec' down_revision = 'ad16b3bd8cb6' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py b/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py index d6f27a06268..b27a4353e8f 100644 --- a/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py +++ b/ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """068 Add package extras index Revision ID: e33a5f2b2a84 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e33a5f2b2a84' down_revision = '266c110eafec' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index( 'idx_package_extra_package_id', 'package_extra_revision', ['package_id', 'current'] diff --git a/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py b/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py index 1137d615f92..42cc39528f7 100644 --- a/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py +++ b/ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """069 Resource url and metadata_modified Revision ID: e7524c675cdb @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e7524c675cdb' down_revision = 'e33a5f2b2a84' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column('resource', sa.Column('url_type', sa.UnicodeText)) op.add_column('resource_revision', sa.Column('url_type', sa.UnicodeText)) diff --git a/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py b/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py index 3498045a108..745d2a39ed6 100644 --- a/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py +++ b/ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """070 Add activity and resource_indexes Revision ID: cfb544112fa7 @@ -6,7 +7,7 @@ """ from alembic import op -import sqlalchemy as sa +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'cfb544112fa7' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index( 'idx_activity_user_id', 'activity', ['user_id', 'timestamp'] ) diff --git a/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py b/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py index 930a826b23b..0f272f8a450 100644 --- a/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py +++ b/ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """071 Add state column to_user_table Revision ID: c16f081ef73a @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'c16f081ef73a' down_revision = 'cfb544112fa7' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'user', sa.Column( diff --git a/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py b/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py index 45d992e7821..fa722b2fd54 100644 --- a/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py +++ b/ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """072 Add resource view Revision ID: 08dcb9233ad7 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '08dcb9233ad7' down_revision = 'c16f081ef73a' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_table( 'resource_view', sa.Column('id', sa.UnicodeText, nullable=False), diff --git a/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py b/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py index b8770af40b1..fa288586d27 100644 --- a/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py +++ b/ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """073 Update resource view resource_id_constraint Revision ID: 011f51208be3 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '011f51208be3' down_revision = '08dcb9233ad7' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_constraint('resource_view_resource_id_fkey', 'resource_view') op.create_foreign_key( 'resource_view_resource_id_fkey', diff --git a/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py b/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py index 07dc9107fd3..0a4c5b1e6ad 100644 --- a/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py +++ b/ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """074 Remove resource groups Revision ID: a4ca55f0f45e @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'a4ca55f0f45e' down_revision = '011f51208be3' @@ -35,6 +36,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'resource', sa.Column( diff --git a/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py b/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py index 3e6b766e068..f26a92152c2 100644 --- a/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py +++ b/ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """075 Rename view plugins Revision ID: 9cdc88c8896a @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '9cdc88c8896a' down_revision = 'a4ca55f0f45e' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py b/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py index c9e178da24d..df891af17bd 100644 --- a/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py +++ b/ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """076 Rename view plugins 2 Revision ID: 59995aa965c0 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '59995aa965c0' down_revision = '9cdc88c8896a' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py b/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py index bc23b1dbf04..73157e417a5 100644 --- a/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py +++ b/ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """077 Add revisions to system_info Revision ID: 51171a04d86d @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '51171a04d86d' down_revision = '59995aa965c0' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'system_info', sa.Column( diff --git a/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py b/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py index 8fa573e85c0..a3c79c4601f 100644 --- a/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py +++ b/ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """078 Remove old authz model Revision ID: ae821876532a @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'ae821876532a' down_revision = '51171a04d86d' @@ -25,7 +26,10 @@ ('idx_ra_role_action', 'role_action', ['action', 'role']), ) + def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_table('role_action') op.drop_table('package_role') op.drop_table('group_role') diff --git a/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py b/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py index c947363382e..d1690c95469 100644 --- a/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py +++ b/ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """079 Resource revision index Revision ID: e0177a15d2c9 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'e0177a15d2c9' down_revision = 'ae821876532a' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index( 'idx_resource_continuity_id', 'resource_revision', ['continuity_id'] ) diff --git a/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py b/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py index 76252c2cd8e..b70ed56ef93 100644 --- a/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py +++ b/ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """080 Continuity id indexes Revision ID: 8224d872c64f @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '8224d872c64f' down_revision = 'e0177a15d2c9' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index( 'idx_member_continuity_id', 'member_revision', ['continuity_id'] ) diff --git a/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py b/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py index bb0ee315ff7..b17b4b6fda1 100644 --- a/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py +++ b/ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """081 Set datastore active Revision ID: a64cf4a79182 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'a64cf4a79182' @@ -17,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return pass diff --git a/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py b/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py index bb1bf48258b..8fcf2777c3f 100644 --- a/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py +++ b/ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """082 Create index creator user_id Revision ID: 8ea886d0ede4 @@ -6,8 +7,7 @@ """ from alembic import op -import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '8ea886d0ede4' down_revision = 'a64cf4a79182' @@ -16,6 +16,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.create_index( 'idx_package_creator_user_id', 'package', ['creator_user_id'] ) diff --git a/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py b/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py index 375c1b73743..8321f7943ab 100644 --- a/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py +++ b/ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """083 Remove related items Revision ID: f98d8fa2a7f7 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'f98d8fa2a7f7' down_revision = '8ea886d0ede4' @@ -28,6 +29,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return conn = op.get_bind() existing = conn.execute("SELECT COUNT(*) FROM related;").fetchone() if existing[0] > 0: diff --git a/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py b/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py index 35a17a6a223..7e78c915f16 100644 --- a/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py +++ b/ckan/migration/versions/084_d85ce5783688_add_metadata_created.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """084 Add metadata created Revision ID: d85ce5783688 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'd85ce5783688' down_revision = 'f98d8fa2a7f7' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.add_column( 'package_revision', sa.Column('metadata_created', sa.TIMESTAMP) ) diff --git a/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py b/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py index c6ba60df91d..3a7787b3409 100644 --- a/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py +++ b/ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """085 Adjust activity timestamps Revision ID: f9bf3d5c4b4d @@ -8,7 +9,7 @@ import datetime from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'f9bf3d5c4b4d' down_revision = 'd85ce5783688' @@ -17,6 +18,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return magic_timestamp = datetime.datetime(2016, 6, 20).toordinal() utc_date = datetime.datetime.utcfromtimestamp(magic_timestamp) diff --git a/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py b/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py index 00a110a9a17..9106d9e322b 100644 --- a/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py +++ b/ckan/migration/versions/086_19663581b3bb_drop_openid_column.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """086 Drop openid column Revision ID: 19663581b3bb @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = '19663581b3bb' down_revision = 'f9bf3d5c4b4d' @@ -16,6 +17,8 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_column('user', 'openid') diff --git a/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py b/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py index 129cd84b225..055efa47b98 100644 --- a/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py +++ b/ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py @@ -1,3 +1,4 @@ +# encoding: utf-8 """087 Remove old authorization tables Revision ID: ff1b303cab77 @@ -7,7 +8,7 @@ """ from alembic import op import sqlalchemy as sa - +from ckan.migration import skip_based_on_legacy_engine_version # revision identifiers, used by Alembic. revision = 'ff1b303cab77' @@ -17,9 +18,12 @@ def upgrade(): + if skip_based_on_legacy_engine_version(op, __name__): + return op.drop_table('authorization_group_user') op.drop_table('authorization_group') + def downgrade(): op.create_table( 'authorization_group', @@ -42,6 +46,5 @@ def downgrade(): sa.UnicodeText, sa.ForeignKey('user.id'), nullable=False - ), - sa.Column('id', sa.UnicodeText, primary_key=True) + ), sa.Column('id', sa.UnicodeText, primary_key=True) ) diff --git a/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py b/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py new file mode 100644 index 00000000000..84b34cb2cc4 --- /dev/null +++ b/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py @@ -0,0 +1,23 @@ +# encoding: utf-8 +"""Remove migrate version table + +Revision ID: 3ad397f70903 +Revises: ff1b303cab77 +Create Date: 2018-09-18 18:16:50.083513 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '3ad397f70903' +down_revision = 'ff1b303cab77' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute('DROP TABLE IF EXISTS migrate_version') + + +def downgrade(): + pass diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 714d779c868..22152b638bf 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -5,12 +5,21 @@ import re from datetime import datetime from time import sleep +from os.path import splitext from six import text_type import vdm.sqlalchemy from vdm.sqlalchemy.base import SQLAlchemySession from sqlalchemy import MetaData, __version__ as sqav, Table from sqlalchemy.util import OrderedDict +from sqlalchemy.exc import ProgrammingError + +from alembic.command import ( + upgrade as alembic_upgrade, + downgrade as alembic_downgrade, + current as alembic_current +) +from alembic.config import Config as AlembicConfig import meta from meta import ( @@ -136,14 +145,11 @@ import ckan.migration -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) DB_CONNECT_RETRIES = 10 -# set up in init_model after metadata is bound -version_table = None - def init_model(engine): '''Call me before using any of the tables or classes in the model''' @@ -156,8 +162,7 @@ def init_model(engine): import sqlalchemy.exc for i in reversed(range(DB_CONNECT_RETRIES)): try: - global version_table - version_table = Table('migrate_version', meta.metadata, autoload=True) + Table('alembic_version', meta.metadata, autoload=True) break except sqlalchemy.exc.NoSuchTableError: break @@ -169,7 +174,8 @@ def init_model(engine): class Repository(vdm.sqlalchemy.Repository): - migrate_repository = ckan.migration.__path__[0] + _repo_path, _dot_repo_name = splitext(ckan.migration.__name__) + migrate_repository = _repo_path + ':' + _dot_repo_name[1:] # note: tables_created value is not sustained between instantiations # so only useful for tests. The alternative is to use @@ -217,11 +223,6 @@ def create_db(self): self.metadata.create_all(bind=self.metadata.bind) log.info('Database tables created') - def latest_migration_version(self): - import migrate.versioning.api as mig - version = mig.version(self.migrate_repository) - return version - def rebuild_db(self): '''Clean and init the db''' if self.tables_created_and_initialised: @@ -245,44 +246,80 @@ def delete_all(self): else: tables = reversed(self.metadata.sorted_tables) for table in tables: - if table.name == 'migrate_version': + if table.name == 'alembic_version': continue connection.execute('delete from "%s"' % table.name) self.session.commit() log.info('Database table data deleted') - def setup_migration_version_control(self, version=None): - import migrate.exceptions - import migrate.versioning.api as mig - # set up db version control (if not already) + def reset_alembic_output(self): + self._alembic_output = [] + + def add_alembic_output(self, *args): + self._alembic_output.append(args) + + def take_alembic_output(self, with_reset=True): + output = self._alembic_output + self._alembic_config = [] + return output + + def setup_migration_version_control(self): + self.reset_alembic_output() + alembic_config = AlembicConfig() + alembic_config.set_main_option( + "script_location", self.migrate_repository + ) + alembic_config.set_main_option( + "sqlalchemy.url", str(self.metadata.bind.url) + ) try: - mig.version_control(self.metadata.bind, - self.migrate_repository, version) - except migrate.exceptions.DatabaseAlreadyControlledError: - pass + sqlalchemy_migrate_version = self.metadata.bind.execute( + u'select version from migrate_version' + ).scalar() + except ProgrammingError: + sqlalchemy_migrate_version = 0 + alembic_config.set_main_option( + "sqlalchemy_migrate_version", str(sqlalchemy_migrate_version) + ) + alembic_config.print_stdout = self.add_alembic_output + + self.alembic_config = alembic_config - def upgrade_db(self, version=None): + def downgrade_db(self, version='base'): + self.setup_migration_version_control() + alembic_downgrade(self.alembic_config, version) + log.info(u'CKAN database version set to: %s', version) + + def upgrade_db(self, version='head'): '''Upgrade db using sqlalchemy migrations. @param version: version to upgrade to (if None upgrade to latest) ''' - assert meta.engine.name in ('postgres', 'postgresql'), \ - 'Database migration - only Postgresql engine supported (not %s).' \ - % meta.engine.name - import migrate.versioning.api as mig + _assert_engine_msg = ( + u'Database migration - only Postgresql engine supported (not %s).' + ) % meta.engine.name + assert meta.engine.name in ( + u'postgres', u'postgresql' + ), _assert_engine_msg self.setup_migration_version_control() - version_before = mig.db_version(self.metadata.bind, self.migrate_repository) - mig.upgrade(self.metadata.bind, self.migrate_repository, version=version) - version_after = mig.db_version(self.metadata.bind, self.migrate_repository) + try: + alembic_current(self.alembic_config) + version_before = self.take_alembic_output()[0][0] + except (TypeError, IndexError): + # alembic is not initialized yet + version_before = 'base' + alembic_upgrade(self.alembic_config, version) + alembic_current(self.alembic_config) + + version_after = self.take_alembic_output()[0][0] if version_after != version_before: - log.info('CKAN database version upgraded: %s -> %s', version_before, version_after) + log.info( + u'CKAN database version upgraded: %s -> %s', + version_before, + version_after + ) else: - log.info('CKAN database version remains as: %s', version_after) - - ##this prints the diffs in a readable format - ##import pprint - ##from migrate.versioning.schemadiff import getDiffOfModelAgainstDatabase - ##pprint.pprint(getDiffOfModelAgainstDatabase(self.metadata, self.metadata.bind).colDiffs) + log.info(u'CKAN database version remains as: %s', version_after) def are_tables_created(self): meta.metadata = MetaData(self.metadata.bind) diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index 871a75d84a3..0fb255ee153 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -229,93 +229,31 @@ class TestImportStar(object): # import * is bad for many reasons and should be avoided. IMPORT_STAR_BLACKLIST_FILES = [ - 'ckan/migration/versions/001_add_existing_tables.py', - 'ckan/migration/versions/002_add_author_and_maintainer.py', - 'ckan/migration/versions/003_add_user_object.py', - 'ckan/migration/versions/004_add_group_object.py', - 'ckan/migration/versions/005_add_authorization_tables.py', - 'ckan/migration/versions/006_add_ratings.py', - 'ckan/migration/versions/007_add_system_roles.py', - 'ckan/migration/versions/008_update_vdm_ids.py', - 'ckan/migration/versions/009_add_creation_timestamps.py', - 'ckan/migration/versions/010_add_user_about.py', - 'ckan/migration/versions/011_add_package_search_vector.py', - 'ckan/migration/versions/012_add_resources.py', - 'ckan/migration/versions/013_add_hash.py', - 'ckan/migration/versions/014_hash_2.py', - 'ckan/migration/versions/015_remove_state_object.py', - 'ckan/migration/versions/016_uuids_everywhere.py', - 'ckan/migration/versions/017_add_pkg_relationships.py', - 'ckan/migration/versions/018_adjust_licenses.py', - 'ckan/migration/versions/019_pkg_relationships_state.py', - 'ckan/migration/versions/020_add_changeset.py', - 'ckan/migration/versions/022_add_group_extras.py', - 'ckan/migration/versions/023_add_harvesting.py', - 'ckan/migration/versions/024_add_harvested_document.py', - 'ckan/migration/versions/025_add_authorization_groups.py', - 'ckan/migration/versions/026_authorization_group_user_pk.py', - 'ckan/migration/versions/027_adjust_harvester.py', - 'ckan/migration/versions/028_drop_harvest_source_status.py', - 'ckan/migration/versions/029_version_groups.py', - 'ckan/migration/versions/030_additional_user_attributes.py', - 'ckan/migration/versions/031_move_openid_to_new_field.py', - 'ckan/migration/versions/032_add_extra_info_field_to_resources.py', - 'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', - 'ckan/migration/versions/034_resource_group_table.py', - 'ckan/migration/versions/035_harvesting_doc_versioning.py', - 'ckan/migration/versions/036_lockdown_roles.py', - 'ckan/migration/versions/037_role_anon_editor.py', - 'ckan/migration/versions/038_delete_migration_tables.py', - 'ckan/migration/versions/039_add_expired_id_and_dates.py', - 'ckan/migration/versions/040_reset_key_on_user.py', - 'ckan/migration/versions/041_resource_new_fields.py', - 'ckan/migration/versions/042_user_revision_indexes.py', - 'ckan/migration/versions/043_drop_postgres_search.py', - 'ckan/migration/versions/044_add_task_status.py', - 'ckan/migration/versions/045_user_name_unique.py', - 'ckan/migration/versions/046_drop_changesets.py', - 'ckan/migration/versions/047_rename_package_group_member.py', - 'ckan/migration/versions/048_add_activity_streams_tables.py', - 'ckan/migration/versions/049_add_group_approval_status.py', - 'ckan/migration/versions/050_term_translation_table.py', - 'ckan/migration/versions/051_add_tag_vocabulary.py', - 'ckan/migration/versions/052_update_member_capacities.py', - 'ckan/migration/versions/053_add_group_logo.py', - 'ckan/migration/versions/056_add_related_table.py', - 'ckan/migration/versions/057_tracking.py', - 'ckan/migration/versions/058_add_follower_tables.py', - 'ckan/migration/versions/059_add_related_count_and_flag.py', - 'ckan/migration/versions/060_add_system_info_table.py', - 'ckan/migration/versions/061_add_follower__group_table.py', - 'ckan/migration/versions/062_add_dashboard_table.py', - 'ckan/migration/versions/063_org_changes.py', - 'ckan/migration/versions/064_add_email_last_sent_column.py', - 'ckan/migration/versions/065_add_email_notifications_preference.py', - 'ckan/plugins/__init__.py', - 'ckan/tests/legacy/functional/api/base.py', - 'ckan/tests/legacy/functional/api/test_api.py', - 'ckan/tests/legacy/functional/api/test_misc.py', - 'ckan/tests/legacy/functional/api/test_package_search.py', - 'ckan/tests/legacy/functional/api/test_resource_search.py', - 'ckan/tests/legacy/functional/api/test_revision_search.py', - 'ckan/tests/legacy/functional/test_group.py', - 'ckan/tests/legacy/functional/test_home.py', - 'ckan/tests/legacy/functional/test_package.py', - 'ckan/tests/legacy/functional/test_package_relationships.py', - 'ckan/tests/legacy/functional/test_tag.py', - 'ckan/tests/legacy/lib/test_helpers.py', - 'ckan/tests/legacy/lib/test_resource_search.py', - 'ckan/tests/legacy/lib/test_tag_search.py', - 'ckan/tests/legacy/misc/test_sync.py', - 'ckan/tests/legacy/models/test_extras.py', - 'ckan/tests/legacy/models/test_misc.py', - 'ckan/tests/legacy/models/test_package.py', - 'ckan/tests/legacy/models/test_package_relationships.py', - 'ckan/tests/legacy/models/test_purge_revision.py', - 'ckan/tests/legacy/models/test_resource.py', - 'ckan/tests/legacy/models/test_revision.py', - 'ckan/tests/legacy/models/test_user.py', - 'fabfile.py', + u'ckan/plugins/__init__.py', + u'ckan/tests/legacy/functional/api/base.py', + u'ckan/tests/legacy/functional/api/test_api.py', + u'ckan/tests/legacy/functional/api/test_misc.py', + u'ckan/tests/legacy/functional/api/test_package_search.py', + u'ckan/tests/legacy/functional/api/test_resource_search.py', + u'ckan/tests/legacy/functional/api/test_revision_search.py', + u'ckan/tests/legacy/functional/test_group.py', + u'ckan/tests/legacy/functional/test_home.py', + u'ckan/tests/legacy/functional/test_package.py', + u'ckan/tests/legacy/functional/test_package_relationships.py', + u'ckan/tests/legacy/functional/test_tag.py', + u'ckan/tests/legacy/lib/test_helpers.py', + u'ckan/tests/legacy/lib/test_resource_search.py', + u'ckan/tests/legacy/lib/test_tag_search.py', + u'ckan/tests/legacy/misc/test_sync.py', + u'ckan/tests/legacy/models/test_extras.py', + u'ckan/tests/legacy/models/test_misc.py', + u'ckan/tests/legacy/models/test_package.py', + u'ckan/tests/legacy/models/test_package_relationships.py', + u'ckan/tests/legacy/models/test_purge_revision.py', + u'ckan/tests/legacy/models/test_resource.py', + u'ckan/tests/legacy/models/test_revision.py', + u'ckan/tests/legacy/models/test_user.py', + u'fabfile.py', ] fails = {} passes = [] @@ -414,71 +352,6 @@ class TestPep8(object): 'ckan/logic/auth/update.py', 'ckan/logic/converters.py', 'ckan/logic/validators.py', - 'ckan/migration/versions/001_add_existing_tables.py', - 'ckan/migration/versions/002_add_author_and_maintainer.py', - 'ckan/migration/versions/003_add_user_object.py', - 'ckan/migration/versions/004_add_group_object.py', - 'ckan/migration/versions/005_add_authorization_tables.py', - 'ckan/migration/versions/006_add_ratings.py', - 'ckan/migration/versions/007_add_system_roles.py', - 'ckan/migration/versions/008_update_vdm_ids.py', - 'ckan/migration/versions/009_add_creation_timestamps.py', - 'ckan/migration/versions/010_add_user_about.py', - 'ckan/migration/versions/011_add_package_search_vector.py', - 'ckan/migration/versions/012_add_resources.py', - 'ckan/migration/versions/013_add_hash.py', - 'ckan/migration/versions/014_hash_2.py', - 'ckan/migration/versions/015_remove_state_object.py', - 'ckan/migration/versions/016_uuids_everywhere.py', - 'ckan/migration/versions/017_add_pkg_relationships.py', - 'ckan/migration/versions/018_adjust_licenses.py', - 'ckan/migration/versions/019_pkg_relationships_state.py', - 'ckan/migration/versions/020_add_changeset.py', - 'ckan/migration/versions/022_add_group_extras.py', - 'ckan/migration/versions/023_add_harvesting.py', - 'ckan/migration/versions/024_add_harvested_document.py', - 'ckan/migration/versions/025_add_authorization_groups.py', - 'ckan/migration/versions/026_authorization_group_user_pk.py', - 'ckan/migration/versions/027_adjust_harvester.py', - 'ckan/migration/versions/028_drop_harvest_source_status.py', - 'ckan/migration/versions/029_version_groups.py', - 'ckan/migration/versions/030_additional_user_attributes.py', - 'ckan/migration/versions/031_move_openid_to_new_field.py', - 'ckan/migration/versions/032_add_extra_info_field_to_resources.py', - 'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', - 'ckan/migration/versions/034_resource_group_table.py', - 'ckan/migration/versions/035_harvesting_doc_versioning.py', - 'ckan/migration/versions/036_lockdown_roles.py', - 'ckan/migration/versions/037_role_anon_editor.py', - 'ckan/migration/versions/038_delete_migration_tables.py', - 'ckan/migration/versions/039_add_expired_id_and_dates.py', - 'ckan/migration/versions/040_reset_key_on_user.py', - 'ckan/migration/versions/041_resource_new_fields.py', - 'ckan/migration/versions/042_user_revision_indexes.py', - 'ckan/migration/versions/043_drop_postgres_search.py', - 'ckan/migration/versions/044_add_task_status.py', - 'ckan/migration/versions/045_user_name_unique.py', - 'ckan/migration/versions/046_drop_changesets.py', - 'ckan/migration/versions/047_rename_package_group_member.py', - 'ckan/migration/versions/048_add_activity_streams_tables.py', - 'ckan/migration/versions/049_add_group_approval_status.py', - 'ckan/migration/versions/050_term_translation_table.py', - 'ckan/migration/versions/051_add_tag_vocabulary.py', - 'ckan/migration/versions/052_update_member_capacities.py', - 'ckan/migration/versions/053_add_group_logo.py', - 'ckan/migration/versions/054_add_resource_created_date.py', - 'ckan/migration/versions/055_update_user_and_activity_detail.py', - 'ckan/migration/versions/056_add_related_table.py', - 'ckan/migration/versions/057_tracking.py', - 'ckan/migration/versions/058_add_follower_tables.py', - 'ckan/migration/versions/059_add_related_count_and_flag.py', - 'ckan/migration/versions/060_add_system_info_table.py', - 'ckan/migration/versions/061_add_follower__group_table.py', - 'ckan/migration/versions/062_add_dashboard_table.py', - 'ckan/migration/versions/063_org_changes.py', - 'ckan/migration/versions/064_add_email_last_sent_column.py', - 'ckan/migration/versions/065_add_email_notifications_preference.py', - 'ckan/migration/versions/067_turn_extras_to_strings.py', 'ckan/misc.py', 'ckan/model/__init__.py', 'ckan/model/activity.py', diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 1c4252c046e..3bf80090219 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -315,89 +315,94 @@ def find_unprefixed_string_literals(filename): u'ckan/logic/schema.py', u'ckan/logic/validators.py', u'ckan/migration/manage.py', - u'ckan/migration/versions/001_add_existing_tables.py', - u'ckan/migration/versions/002_add_author_and_maintainer.py', - u'ckan/migration/versions/003_add_user_object.py', - u'ckan/migration/versions/004_add_group_object.py', - u'ckan/migration/versions/005_add_authorization_tables.py', - u'ckan/migration/versions/006_add_ratings.py', - u'ckan/migration/versions/007_add_system_roles.py', - u'ckan/migration/versions/008_update_vdm_ids.py', - u'ckan/migration/versions/009_add_creation_timestamps.py', - u'ckan/migration/versions/010_add_user_about.py', - u'ckan/migration/versions/011_add_package_search_vector.py', - u'ckan/migration/versions/012_add_resources.py', - u'ckan/migration/versions/013_add_hash.py', - u'ckan/migration/versions/014_hash_2.py', - u'ckan/migration/versions/015_remove_state_object.py', - u'ckan/migration/versions/016_uuids_everywhere.py', - u'ckan/migration/versions/017_add_pkg_relationships.py', - u'ckan/migration/versions/018_adjust_licenses.py', - u'ckan/migration/versions/019_pkg_relationships_state.py', - u'ckan/migration/versions/020_add_changeset.py', - u'ckan/migration/versions/022_add_group_extras.py', - u'ckan/migration/versions/023_add_harvesting.py', - u'ckan/migration/versions/024_add_harvested_document.py', - u'ckan/migration/versions/025_add_authorization_groups.py', - u'ckan/migration/versions/026_authorization_group_user_pk.py', - u'ckan/migration/versions/027_adjust_harvester.py', - u'ckan/migration/versions/028_drop_harvest_source_status.py', - u'ckan/migration/versions/029_version_groups.py', - u'ckan/migration/versions/030_additional_user_attributes.py', - u'ckan/migration/versions/031_move_openid_to_new_field.py', - u'ckan/migration/versions/032_add_extra_info_field_to_resources.py', - u'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', - u'ckan/migration/versions/034_resource_group_table.py', - u'ckan/migration/versions/035_harvesting_doc_versioning.py', - u'ckan/migration/versions/036_lockdown_roles.py', - u'ckan/migration/versions/037_role_anon_editor.py', - u'ckan/migration/versions/038_delete_migration_tables.py', - u'ckan/migration/versions/039_add_expired_id_and_dates.py', - u'ckan/migration/versions/040_reset_key_on_user.py', - u'ckan/migration/versions/041_resource_new_fields.py', - u'ckan/migration/versions/042_user_revision_indexes.py', - u'ckan/migration/versions/043_drop_postgres_search.py', - u'ckan/migration/versions/044_add_task_status.py', - u'ckan/migration/versions/045_user_name_unique.py', - u'ckan/migration/versions/046_drop_changesets.py', - u'ckan/migration/versions/047_rename_package_group_member.py', - u'ckan/migration/versions/048_add_activity_streams_tables.py', - u'ckan/migration/versions/049_add_group_approval_status.py', - u'ckan/migration/versions/050_term_translation_table.py', - u'ckan/migration/versions/051_add_tag_vocabulary.py', - u'ckan/migration/versions/052_update_member_capacities.py', - u'ckan/migration/versions/053_add_group_logo.py', - u'ckan/migration/versions/054_add_resource_created_date.py', - u'ckan/migration/versions/055_update_user_and_activity_detail.py', - u'ckan/migration/versions/056_add_related_table.py', - u'ckan/migration/versions/057_tracking.py', - u'ckan/migration/versions/058_add_follower_tables.py', - u'ckan/migration/versions/059_add_related_count_and_flag.py', - u'ckan/migration/versions/060_add_system_info_table.py', - u'ckan/migration/versions/061_add_follower__group_table.py', - u'ckan/migration/versions/062_add_dashboard_table.py', - u'ckan/migration/versions/063_org_changes.py', - u'ckan/migration/versions/064_add_email_last_sent_column.py', - u'ckan/migration/versions/065_add_email_notifications_preference.py', - u'ckan/migration/versions/066_default_package_type.py', - u'ckan/migration/versions/067_turn_extras_to_strings.py', - u'ckan/migration/versions/068_add_package_extras_index.py', - u'ckan/migration/versions/069_resource_url_and_metadata_modified.py', - u'ckan/migration/versions/070_add_activity_and_resource_indexes.py', - u'ckan/migration/versions/071_add_state_column_to_user_table.py', - u'ckan/migration/versions/072_add_resource_view.py', - u'ckan/migration/versions/073_update_resource_view_resource_id_constraint.py', - u'ckan/migration/versions/074_remove_resource_groups.py', - u'ckan/migration/versions/075_rename_view_plugins.py', - u'ckan/migration/versions/076_rename_view_plugins_2.py', - u'ckan/migration/versions/077_add_revisions_to_system_info.py', - u'ckan/migration/versions/078_remove_old_authz_model.py', - u'ckan/migration/versions/079_resource_revision_index.py', - u'ckan/migration/versions/080_continuity_id_indexes.py', - u'ckan/migration/versions/081_set_datastore_active.py', - u'ckan/migration/versions/082_create_index_creator_user_id.py', - u'ckan/migration/versions/083_remove_related_items.py', - u'ckan/migration/versions/084_add_metadata_created.py', + u'ckan/migration/versions/001_103676e0a497_create_existing_tables.py', + u'ckan/migration/versions/002_86fdd8c54775_add_author_and_maintainer.py', + u'ckan/migration/versions/003_f22b4f5241a5_add_user_object.py', + u'ckan/migration/versions/004_f92ee205e46d_add_group_object.py', + u'ckan/migration/versions/005_12c2232c15f5_add_authorization_tables.py', + u'ckan/migration/versions/006_c83955e7acb6_add_ratings.py', + u'ckan/migration/versions/007_1928d4af1cda_add_system_roles.py', + u'ckan/migration/versions/008_e8283ffb257e_update_vdm_ids.py', + u'ckan/migration/versions/009_b739a48de5c4_add_creation_timestamps.py', + u'ckan/migration/versions/010_a6f13bf14d0c_add_user_about.py', + u'ckan/migration/versions/011_866f6370b4ac_add_package_search_vector.py', + u'ckan/migration/versions/012_e5ca33a5d445_add_resources.py', + u'ckan/migration/versions/013_8a3a5af39797_add_hash.py', + u'ckan/migration/versions/014_93519b684820_hash_2.py', + u'ckan/migration/versions/015_6d8ffebcaf54_remove_state_object.py', + u'ckan/migration/versions/016_37ada738328e_uuids_everywhere.py', + u'ckan/migration/versions/017_1250b2ff3e36_add_pkg_relationships.py', + u'ckan/migration/versions/018_05a0778051ca_adjust_licenses.py', + u'ckan/migration/versions/019_b2eb6f34a638_pkg_relationships_state.py', + u'ckan/migration/versions/020_69a0b0efc609_add_changeset.py', + u'ckan/migration/versions/021_765143af2ba3_postgresql_upgrade_sql.py', + u'ckan/migration/versions/022_7b324ca6c0dc_add_group_extras.py', + u'ckan/migration/versions/023_87fdd05f0744_add_harvesting.py', + u'ckan/migration/versions/024_12981fe12484_add_harvested_document.py', + u'ckan/migration/versions/025_b581622ad327_add_authorization_groups.py', + u'ckan/migration/versions/026_3615b25af443_authorization_group_user_pk.py', + u'ckan/migration/versions/027_11e5745c6fc9_adjust_harvester.py', + u'ckan/migration/versions/028_cdd68fe9ba21_drop_harvest_source_status.py', + u'ckan/migration/versions/029_1bfdf4240915_version_groups.py', + u'ckan/migration/versions/030_b16cbf164c8a_additional_user_attributes.py', + u'ckan/migration/versions/031_1b05245167d6_move_openid_to_new_field.py', + u'ckan/migration/versions/032_d89e0731422d_add_extra_info_field_to_resources.py', + u'ckan/migration/versions/033_6da92ef2df15_auth_group_user_id_add_conditional.py', + u'ckan/migration/versions/034_6c600693af5b_resource_group_table.py', + u'ckan/migration/versions/035_81148ccebd6c_harvesting_doc_versioning.py', + u'ckan/migration/versions/036_ecaa8b38782f_lockdown_roles.py', + u'ckan/migration/versions/037_edcf3b8c3c1b_role_anon_editor.py', + u'ckan/migration/versions/038_fd6622e3d964_delete_migration_tables.py', + u'ckan/migration/versions/039_cca459c76d45_add_expired_id_and_dates.py', + u'ckan/migration/versions/040_500a08f4818e_reset_key_on_user.py', + u'ckan/migration/versions/041_6817d4e3bdc3_resource_new_fields.py', + u'ckan/migration/versions/042_da65e2877034_user_revision_indexes.py', + u'ckan/migration/versions/043_bd38cd6502b2_drop_postgres_search.py', + u'ckan/migration/versions/044_4190eeeb8d73_add_task_status.py', + u'ckan/migration/versions/045_54e3f155d945_user_name_unique.py', + u'ckan/migration/versions/046_b69e9b80396f_drop_changesets.py', + u'ckan/migration/versions/047_883a7c406926_rename_package_group_member.py', + u'ckan/migration/versions/048_4a7011172b3f_add_activity_streams_tables.py', + u'ckan/migration/versions/049_e0c06c2177b5_add_group_approval_status.py', + u'ckan/migration/versions/050_01a6b058cb7f_term_translation_table.py', + u'ckan/migration/versions/051_a4fb0d85ced6_add_tag_vocabulary.py', + u'ckan/migration/versions/052_ba693d64c6d7_update_member_capacities.py', + u'ckan/migration/versions/053_9d051a099097_add_group_logo.py', + u'ckan/migration/versions/054_da21b38da4db_add_resource_created_date.py', + u'ckan/migration/versions/055_048f7db947bf_update_user_and_activity_detail.py', + u'ckan/migration/versions/056_11af3215ae89_add_related_table.py', + u'ckan/migration/versions/057_660a5aae527e_tracking.py', + u'ckan/migration/versions/058_bd36d1826a5d_add_follower_tables.py', + u'ckan/migration/versions/059_9291bb46f352_add_related_count_and_flag.py', + u'ckan/migration/versions/060_31ad11c518fc_add_system_info_table.py', + u'ckan/migration/versions/061_338d460bc460_add_follower_group_table.py', + u'ckan/migration/versions/062_6deb2bbab394_add_dashboard_table.py', + u'ckan/migration/versions/063_8b633852cb7a_org_changes.py', + u'ckan/migration/versions/064_4f8becd4919a_add_email_last_sent_column.py', + u'ckan/migration/versions/065_1fab0bc6439e_add_email_notifications_preference.py', + u'ckan/migration/versions/066_ad16b3bd8cb6_default_package_type.py', + u'ckan/migration/versions/067_266c110eafec_turn_extras_to_strings.py', + u'ckan/migration/versions/068_e33a5f2b2a84_add_package_extras_index.py', + u'ckan/migration/versions/069_e7524c675cdb_resource_url_and_metadata_modified.py', + u'ckan/migration/versions/070_cfb544112fa7_add_activity_and_resource_indexes.py', + u'ckan/migration/versions/071_c16f081ef73a_add_state_column_to_user_table.py', + u'ckan/migration/versions/072_08dcb9233ad7_add_resource_view.py', + u'ckan/migration/versions/073_011f51208be3_update_resource_view_resource_id_.py', + u'ckan/migration/versions/074_a4ca55f0f45e_remove_resource_groups.py', + u'ckan/migration/versions/075_9cdc88c8896a_rename_view_plugins.py', + u'ckan/migration/versions/076_59995aa965c0_rename_view_plugins_2.py', + u'ckan/migration/versions/077_51171a04d86d_add_revisions_to_system_info.py', + u'ckan/migration/versions/078_ae821876532a_remove_old_authz_model.py', + u'ckan/migration/versions/079_e0177a15d2c9_resource_revision_index.py', + u'ckan/migration/versions/080_8224d872c64f_continuity_id_indexes.py', + u'ckan/migration/versions/081_a64cf4a79182_set_datastore_active.py', + u'ckan/migration/versions/082_8ea886d0ede4_create_index_creator_user_id.py', + u'ckan/migration/versions/083_f98d8fa2a7f7_remove_related_items.py', + u'ckan/migration/versions/084_d85ce5783688_add_metadata_created.py', + u'ckan/migration/versions/085_f9bf3d5c4b4d_adjust_activity_timestamps.py', + u'ckan/migration/versions/086_19663581b3bb_drop_openid_column.py', + u'ckan/migration/versions/087_ff1b303cab77_remove_old_authorization_tables.py', + u'ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py', u'ckan/model/__init__.py', u'ckan/model/activity.py', u'ckan/model/core.py', From 1dee1bb323f8608121c255663278775d3e2a4b70 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Fri, 21 Sep 2018 11:29:57 +0300 Subject: [PATCH 3/6] Suggested improvements and comments --- ckan/migration/__init__.py | 14 ++++++++++++++ ckan/migration/alembic.ini | 10 +++++++++- ...88_3ad397f70903_remove_migrate_version_table.py | 13 +++++++++++++ ckan/model/__init__.py | 5 +++++ 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/ckan/migration/__init__.py b/ckan/migration/__init__.py index 912ce6b7da9..1b0bbb92874 100644 --- a/ckan/migration/__init__.py +++ b/ckan/migration/__init__.py @@ -2,6 +2,20 @@ def skip_based_on_legacy_engine_version(op, filename): + u'''Safe way to update instances sqlalchemy-migrate migrations applied. + + CKAN `db upgrade\init` command is trying to obtain current version + of sqlalchemy-migrate migrations from database. In that case, we + are going to compare existing version from DB with alembic + migration script's prefix in filename which defines corresponding + version of sqlalchemy-migrate script. We need this, because + alembic uses string ids instead of incremental numbers for + identifying current migration version. If alembic script's version + is less than version of currently applied sqlalchemy migration, + than it just marked as applied, but no SQL queries will be + actually executed. Thus there are no difference between updating + existing portals and initializing new ones. + ''' conf = op.get_context().config version = conf.get_main_option(u'sqlalchemy_migrate_version') if version: diff --git a/ckan/migration/alembic.ini b/ckan/migration/alembic.ini index eb2da491029..23cd14af58e 100644 --- a/ckan/migration/alembic.ini +++ b/ckan/migration/alembic.ini @@ -35,7 +35,15 @@ script_location = . # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = postgresql://ckan_default:root@localhost/ckan_alembic +# This value does not affects CKAN commands, but developers are able +# to execute alembic commands(i.e. `alembic heads`, `alembic +# revision`) directly from this folder with line bellow. If it +# configured to correct values, migrations can even be applied(but it +# strongly discouraged - use CKAN's upgrade command instead). This +# functionality is considered as internal, so don't rely on it, +# because it may be removed in future. Use official CKAN commands +# rather than direct interactions with alembic +sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default # Logging configuration diff --git a/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py b/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py index 84b34cb2cc4..a4e797de891 100644 --- a/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py +++ b/ckan/migration/versions/088_3ad397f70903_remove_migrate_version_table.py @@ -16,8 +16,21 @@ def upgrade(): + '''Drop version table, created by sqlalchemy-migrate. + + There is a chance, that we are initializing a new instance and + there is no `migrate_version` table, so DO NOT remove `IF EXISTS` + clause. + ''' op.execute('DROP TABLE IF EXISTS migrate_version') def downgrade(): + '''We aren't going to recreate `migrate_version` here. + + There is a chance, that this table even never was created for + target database. This migration tries to seamlessly upgrade + existing instance from usage of sqlalchemy-migrate to alembic. And + we don't want to downgrade to sqlalchemy-migrate back again. + ''' pass diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 22152b638bf..6136bf3dcb0 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -278,9 +278,14 @@ def setup_migration_version_control(self): ).scalar() except ProgrammingError: sqlalchemy_migrate_version = 0 + + # this value is used for graceful upgrade from + # sqlalchemy-migrate to alembic alembic_config.set_main_option( "sqlalchemy_migrate_version", str(sqlalchemy_migrate_version) ) + # This is an interceptor for alembic output. Otherwise, + # everything will be printed to stdout alembic_config.print_stdout = self.add_alembic_output self.alembic_config = alembic_config From 6468d7c7d524059602ae49d5fee64293e53b9146 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 9 May 2019 14:29:58 +0300 Subject: [PATCH 4/6] Update PR with new migrations and Click CLI --- ckan/cli/db.py | 61 ++++++++++++++----- ..._delete_extrase_which_are_deleted_state.py | 40 ++++++++++++ ...8_delete_extras_which_are_deleted_state.py | 14 ----- ...0926e_package_activity_migration_check.py} | 42 ++++++++++--- ...0dcd44de4b_delete_migrate_version_table.py | 37 +++++++++++ ckan/model/__init__.py | 18 +++--- 6 files changed, 168 insertions(+), 44 deletions(-) create mode 100644 ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py delete mode 100644 ckan/migration/versions/088_delete_extras_which_are_deleted_state.py rename ckan/migration/versions/{089_package_activity_migration_check.py => 089_23c92480926e_package_activity_migration_check.py} (54%) create mode 100644 ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py diff --git a/ckan/cli/db.py b/ckan/cli/db.py index 860668cfd64..fa52434ecfe 100644 --- a/ckan/cli/db.py +++ b/ckan/cli/db.py @@ -1,7 +1,8 @@ # encoding: utf-8 +import os import logging - +import ckan.migration as migration_repo import click from ckan.cli import error_shout @@ -44,8 +45,8 @@ def cleandb(): @db.command(u'upgrade', short_help=u'Upgrade the database') -@click.option(u'-v', u'--version', help=u'Migration version') -def updatedb(version=None): +@click.option(u'-v', u'--version', help=u'Migration version', default=u'head') +def updatedb(version): u'''Upgrading the database''' try: import ckan.model as model @@ -56,17 +57,49 @@ def updatedb(version=None): click.secho(u'Upgrading DB: SUCCESS', fg=u'green', bold=True) -@db.command(u'version', short_help=u'Returns current version of data schema') -def version(): - u'''Return current version''' - log.info(u"Returning current DB version") +@db.command(u'downgrade', short_help=u'Downgrade the database') +@click.option(u'-v', u'--version', help=u'Migration version', default=u'base') +def downgradedb(version): + u'''Upgrading the database''' try: - from ckan.model import Session - ver = Session.execute(u'select version from ' - u'migrate_version;').fetchall() - click.secho( - u"Latest data schema version: {0}".format(ver[0][0]), - bold=True - ) + import ckan.model as model + model.repo.downgrade_db(version) except Exception as e: error_shout(e) + else: + click.secho(u'Downgrading DB: SUCCESS', fg=u'green', bold=True) + + +@db.command(u'version', short_help=u'Returns current version of data schema') +@click.option(u'--hash', is_flag=True) +def version(hash): + u'''Return current version''' + log.info(u"Returning current DB version") + import ckan.model as model + model.repo.setup_migration_version_control() + current = model.repo.current_version() + if not hash: + current = _version_hash_to_ordinal(current) + click.secho( + u'Current DB version: {}'.format(current), + fg=u'green', bold=True + ) + + +def _version_hash_to_ordinal(version): + if u'base' == version: + return 0 + versions_dir = os.path.join( + os.path.dirname(migration_repo.__file__), u'versions' + ) + versions = sorted(os.listdir(versions_dir)) + + # latest version looks like `123abc (head)` + if version.endswith('(head)'): + return int(versions[-1].split(u'_')[0]) + for name in versions: + if version in name: + return int(name.split(u'_')[0]) + error_shout(u'Version `{}` was not found in {}'.format( + version, versions_dir + )) diff --git a/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py b/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py new file mode 100644 index 00000000000..0631759c6a2 --- /dev/null +++ b/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py @@ -0,0 +1,40 @@ +# encoding: utf-8 + +"""delete extrase which are deleted state + +Revision ID: 3537d5420e0e +Revises: ff1b303cab77 +Create Date: 2019-05-09 13:38:22.072361 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '3537d5420e0e' +down_revision = 'ff1b303cab77' +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_constraint( + 'package_extra_revision_continuity_id_fkey', 'package_extra_revision' + ) + op.drop_constraint( + 'group_extra_revision_continuity_id_fkey', 'group_extra_revision' + ) + + conn = op.get_bind() + conn.execute('''DELETE FROM "package_extra" WHERE state='deleted';''') + conn.execute('''DELETE FROM "group_extra" WHERE state='deleted';''') + + +def downgrade(): + op.create_foreign_key( + 'package_extra_revision_continuity_id_fkey', 'package_extra_revision', + 'package_extra', ['continuity_id'], ['id'] + ) + op.create_foreign_key( + 'group_extra_revision_continuity_id_fkey', 'group_extra_revision', + 'group_extra', ['group_id'], ['id'] + ) diff --git a/ckan/migration/versions/088_delete_extras_which_are_deleted_state.py b/ckan/migration/versions/088_delete_extras_which_are_deleted_state.py deleted file mode 100644 index 7cf4a2c3752..00000000000 --- a/ckan/migration/versions/088_delete_extras_which_are_deleted_state.py +++ /dev/null @@ -1,14 +0,0 @@ -# encoding: utf-8 - - -def upgrade(migrate_engine): - migrate_engine.execute( - ''' -ALTER TABLE "package_extra_revision" - DROP CONSTRAINT IF EXISTS package_extra_revision_continuity_id_fkey; -ALTER TABLE "group_extra_revision" - DROP CONSTRAINT IF EXISTS group_extra_revision_continuity_id_fkey; -DELETE FROM "package_extra" WHERE state='deleted'; -DELETE FROM "group_extra" WHERE state='deleted'; - ''' - ) diff --git a/ckan/migration/versions/089_package_activity_migration_check.py b/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py similarity index 54% rename from ckan/migration/versions/089_package_activity_migration_check.py rename to ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py index aed0a797e61..fdd67105247 100644 --- a/ckan/migration/versions/089_package_activity_migration_check.py +++ b/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py @@ -1,14 +1,31 @@ # encoding: utf-8 +"""package activity migration check +Revision ID: 23c92480926e +Revises: 3537d5420e0e +Create Date: 2019-05-09 13:39:17.486611 + +""" import sys +from alembic import op + from ckan.migration.migrate_package_activity import num_unmigrated +# revision identifiers, used by Alembic. +revision = '23c92480926e' +down_revision = '3537d5420e0e' +branch_labels = None +depends_on = None + + +def upgrade(): + conn = op.get_bind() + num_unmigrated_dataset_activities = num_unmigrated(conn) -def upgrade(migrate_engine): - num_unmigrated_dataset_activities = num_unmigrated(migrate_engine) if num_unmigrated_dataset_activities: - print(''' + print( + ''' NOTE: You have {num_unmigrated} unmigrated package activities. @@ -25,14 +42,23 @@ def upgrade(migrate_engine): ckan.auth.public_activity_stream_detail = true More information about all of this is here: - https://github.com/ckan/ckan/wiki/Migrate-package-activity - '''.format(num_unmigrated=num_unmigrated_dataset_activities)) + https://github.com/ckan/ckan/wiki/Migrate-package-activity'''.format( + num_unmigrated=num_unmigrated_dataset_activities + ) + ) else: # there are no unmigrated package activities are_any_datasets = bool( - migrate_engine.execute(u'SELECT id FROM PACKAGE LIMIT 1').rowcount) + conn.execute(u'SELECT id FROM PACKAGE LIMIT 1').rowcount + ) # no need to tell the user if there are no datasets - this could just # be a fresh CKAN install if are_any_datasets: - print(u'You have no unmigrated package activities - you do not ' - 'need to run migrate_package_activity.py.') + print( + u'You have no unmigrated package activities - you do not ' + 'need to run migrate_package_activity.py.' + ) + + +def downgrade(): + pass diff --git a/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py b/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py new file mode 100644 index 00000000000..9e9bfddd23f --- /dev/null +++ b/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py @@ -0,0 +1,37 @@ +# encoding: utf-8 + +"""delete migrate version table + +Revision ID: 980dcd44de4b +Revises: 23c92480926e +Create Date: 2019-05-09 13:39:44.097930 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '980dcd44de4b' +down_revision = '23c92480926e' +branch_labels = None +depends_on = None + + +def upgrade(): + '''Drop version table, created by sqlalchemy-migrate. + + There is a chance, that we are initializing a new instance and + there is no `migrate_version` table, so DO NOT remove `IF EXISTS` + clause. + ''' + op.execute('DROP TABLE IF EXISTS migrate_version') + + +def downgrade(): + '''We aren't going to recreate `migrate_version` here. + + There is a chance, that this table even never was created for + target database. This migration tries to seamlessly upgrade + existing instance from usage of sqlalchemy-migrate to alembic. And + we don't want to downgrade to sqlalchemy-migrate back again. + ''' + pass diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 8c46b3b55cf..3641b81bde2 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -290,6 +290,14 @@ def setup_migration_version_control(self): self.alembic_config = alembic_config + def current_version(self): + try: + alembic_current(self.alembic_config) + return self.take_alembic_output()[0][0] + except (TypeError, IndexError): + # alembic is not initialized yet + return 'base' + def downgrade_db(self, version='base'): self.setup_migration_version_control() alembic_downgrade(self.alembic_config, version) @@ -307,16 +315,10 @@ def upgrade_db(self, version='head'): u'postgres', u'postgresql' ), _assert_engine_msg self.setup_migration_version_control() - try: - alembic_current(self.alembic_config) - version_before = self.take_alembic_output()[0][0] - except (TypeError, IndexError): - # alembic is not initialized yet - version_before = 'base' + version_before = self.current_version() alembic_upgrade(self.alembic_config, version) - alembic_current(self.alembic_config) + version_after = self.current_version() - version_after = self.take_alembic_output()[0][0] if version_after != version_before: log.info( u'CKAN database version upgraded: %s -> %s', From 2751c54788a6b76c54cdf54549473b12a4eb6778 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Thu, 9 May 2019 14:40:30 +0300 Subject: [PATCH 5/6] string literals --- ckan/cli/db.py | 2 +- ckan/migration/__init__.py | 2 +- ..._delete_extrase_which_are_deleted_state.py | 22 +++++++++---------- ...80926e_package_activity_migration_check.py | 6 ++--- ...0dcd44de4b_delete_migrate_version_table.py | 10 ++++----- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ckan/cli/db.py b/ckan/cli/db.py index fa52434ecfe..cbdf7c81439 100644 --- a/ckan/cli/db.py +++ b/ckan/cli/db.py @@ -95,7 +95,7 @@ def _version_hash_to_ordinal(version): versions = sorted(os.listdir(versions_dir)) # latest version looks like `123abc (head)` - if version.endswith('(head)'): + if version.endswith(u'(head)'): return int(versions[-1].split(u'_')[0]) for name in versions: if version in name: diff --git a/ckan/migration/__init__.py b/ckan/migration/__init__.py index 1b0bbb92874..6684dc32332 100644 --- a/ckan/migration/__init__.py +++ b/ckan/migration/__init__.py @@ -4,7 +4,7 @@ def skip_based_on_legacy_engine_version(op, filename): u'''Safe way to update instances sqlalchemy-migrate migrations applied. - CKAN `db upgrade\init` command is trying to obtain current version + CKAN `db upgrade/init` command is trying to obtain current version of sqlalchemy-migrate migrations from database. In that case, we are going to compare existing version from DB with alembic migration script's prefix in filename which defines corresponding diff --git a/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py b/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py index 0631759c6a2..71a0c97ad83 100644 --- a/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py +++ b/ckan/migration/versions/088_3537d5420e0e_delete_extrase_which_are_deleted_state.py @@ -1,5 +1,4 @@ # encoding: utf-8 - """delete extrase which are deleted state Revision ID: 3537d5420e0e @@ -10,31 +9,32 @@ from alembic import op # revision identifiers, used by Alembic. -revision = '3537d5420e0e' -down_revision = 'ff1b303cab77' +revision = u'3537d5420e0e' +down_revision = u'ff1b303cab77' branch_labels = None depends_on = None def upgrade(): op.drop_constraint( - 'package_extra_revision_continuity_id_fkey', 'package_extra_revision' + u'package_extra_revision_continuity_id_fkey', u'package_extra_revision' ) op.drop_constraint( - 'group_extra_revision_continuity_id_fkey', 'group_extra_revision' + u'group_extra_revision_continuity_id_fkey', u'group_extra_revision' ) conn = op.get_bind() - conn.execute('''DELETE FROM "package_extra" WHERE state='deleted';''') - conn.execute('''DELETE FROM "group_extra" WHERE state='deleted';''') + conn.execute(u'''DELETE FROM "package_extra" WHERE state='deleted';''') + conn.execute(u'''DELETE FROM "group_extra" WHERE state='deleted';''') def downgrade(): op.create_foreign_key( - 'package_extra_revision_continuity_id_fkey', 'package_extra_revision', - 'package_extra', ['continuity_id'], ['id'] + u'package_extra_revision_continuity_id_fkey', + u'package_extra_revision', u'package_extra', [u'continuity_id'], + [u'id'] ) op.create_foreign_key( - 'group_extra_revision_continuity_id_fkey', 'group_extra_revision', - 'group_extra', ['group_id'], ['id'] + u'group_extra_revision_continuity_id_fkey', u'group_extra_revision', + u'group_extra', [u'group_id'], [u'id'] ) diff --git a/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py b/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py index fdd67105247..aa77e59180f 100644 --- a/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py +++ b/ckan/migration/versions/089_23c92480926e_package_activity_migration_check.py @@ -13,8 +13,8 @@ from ckan.migration.migrate_package_activity import num_unmigrated # revision identifiers, used by Alembic. -revision = '23c92480926e' -down_revision = '3537d5420e0e' +revision = u'23c92480926e' +down_revision = u'3537d5420e0e' branch_labels = None depends_on = None @@ -25,7 +25,7 @@ def upgrade(): if num_unmigrated_dataset_activities: print( - ''' + u''' NOTE: You have {num_unmigrated} unmigrated package activities. diff --git a/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py b/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py index 9e9bfddd23f..18a942d160f 100644 --- a/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py +++ b/ckan/migration/versions/090_980dcd44de4b_delete_migrate_version_table.py @@ -10,24 +10,24 @@ from alembic import op # revision identifiers, used by Alembic. -revision = '980dcd44de4b' -down_revision = '23c92480926e' +revision = u'980dcd44de4b' +down_revision = u'23c92480926e' branch_labels = None depends_on = None def upgrade(): - '''Drop version table, created by sqlalchemy-migrate. + u'''Drop version table, created by sqlalchemy-migrate. There is a chance, that we are initializing a new instance and there is no `migrate_version` table, so DO NOT remove `IF EXISTS` clause. ''' - op.execute('DROP TABLE IF EXISTS migrate_version') + op.execute(u'DROP TABLE IF EXISTS migrate_version') def downgrade(): - '''We aren't going to recreate `migrate_version` here. + u'''We aren't going to recreate `migrate_version` here. There is a chance, that this table even never was created for target database. This migration tries to seamlessly upgrade From 18101be74298500195460aaa13cb969db5fca6d2 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 10 May 2019 22:20:06 +0300 Subject: [PATCH 6/6] Update ckan/cli/db.py Co-Authored-By: David Read --- ckan/cli/db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/cli/db.py b/ckan/cli/db.py index cbdf7c81439..8a6ab70fccd 100644 --- a/ckan/cli/db.py +++ b/ckan/cli/db.py @@ -60,7 +60,7 @@ def updatedb(version): @db.command(u'downgrade', short_help=u'Downgrade the database') @click.option(u'-v', u'--version', help=u'Migration version', default=u'base') def downgradedb(version): - u'''Upgrading the database''' + u'''Downgrading the database''' try: import ckan.model as model model.repo.downgrade_db(version)