From b54ce7e06b781a530be4f16eb6da8fb766f55f8d Mon Sep 17 00:00:00 2001 From: Darrin Mann Date: Wed, 19 Jul 2017 15:32:12 -0400 Subject: [PATCH 1/3] Test that consistency migration does not update is_deleted projects --- .../with_deleted_project/1_2_4_1_2_1.yml | 65 +++++++++++++++++++ spec/lib/tasks/db_data_migrate_rake_spec.rb | 5 ++ 2 files changed, 70 insertions(+) create mode 100644 spec/cassettes/db_data_migrate/_invoke/consistency_migration/for_project/with_deleted_project/1_2_4_1_2_1.yml diff --git a/spec/cassettes/db_data_migrate/_invoke/consistency_migration/for_project/with_deleted_project/1_2_4_1_2_1.yml b/spec/cassettes/db_data_migrate/_invoke/consistency_migration/for_project/with_deleted_project/1_2_4_1_2_1.yml new file mode 100644 index 000000000..dedebbce7 --- /dev/null +++ b/spec/cassettes/db_data_migrate/_invoke/consistency_migration/for_project/with_deleted_project/1_2_4_1_2_1.yml @@ -0,0 +1,65 @@ +--- +http_interactions: +- request: + method: get + uri: http://swift.local:12345/auth/v1.0 + body: + encoding: US-ASCII + string: '' + headers: + X-Auth-User: + - test:tester + X-Auth-Key: + - testing + response: + status: + code: 200 + message: OK + headers: + X-Storage-Url: + - http://swift.local:12345/v1/AUTH_test + X-Auth-Token: + - AUTH_tkb5fc1ba5ddd9424c9be0d92a7efbac77 + Content-Type: + - text/html; charset=UTF-8 + X-Storage-Token: + - AUTH_tkb5fc1ba5ddd9424c9be0d92a7efbac77 + Content-Length: + - '0' + X-Trans-Id: + - tx045a191ef0054f08bbca0-0059556dfa + Date: + - Thu, 29 Jun 2017 21:15:38 GMT + body: + encoding: UTF-8 + string: '' + http_version: + recorded_at: Thu, 29 Jun 2017 21:15:38 GMT +- request: + method: head + uri: http://swift.local:12345/v1/AUTH_test/69f63381-4674-4257-aa96-e1c72d08fac3 + body: + encoding: US-ASCII + string: '' + headers: + X-Auth-Token: + - AUTH_tkb5fc1ba5ddd9424c9be0d92a7efbac77 + response: + status: + code: 404 + message: Not Found + headers: + Content-Length: + - '0' + Content-Type: + - text/html; charset=UTF-8 + X-Trans-Id: + - txcbba1878646545c794b19-0059556dfa + Date: + - Thu, 29 Jun 2017 21:15:38 GMT + body: + encoding: UTF-8 + string: '' + http_version: + recorded_at: Thu, 29 Jun 2017 21:15:38 GMT +recorded_with: VCR 3.0.3 diff --git a/spec/lib/tasks/db_data_migrate_rake_spec.rb b/spec/lib/tasks/db_data_migrate_rake_spec.rb index 378eab9c3..a3b6ba19c 100644 --- a/spec/lib/tasks/db_data_migrate_rake_spec.rb +++ b/spec/lib/tasks/db_data_migrate_rake_spec.rb @@ -133,6 +133,11 @@ expect(storage_provider.get_container_meta(record.id)).not_to be_nil } it_behaves_like 'a consistency migration', :init_project_storage + + context 'with deleted project' do + before(:each) { FactoryGirl.create(:project, :deleted, is_consistent: nil) } + it { expect {invoke_task}.not_to change{Project.where(is_consistent: nil).count} } + end end context 'for upload' do From 9ae15ac60d37faa667805d80c4f6a51294d810c3 Mon Sep 17 00:00:00 2001 From: Darrin Mann Date: Wed, 19 Jul 2017 15:32:51 -0400 Subject: [PATCH 2/3] Remove unused cassette --- .../should_update_is_consistent_to_true.yml | 67 ------------------- 1 file changed, 67 deletions(-) delete mode 100644 spec/cassettes/db_data_migrate/_invoke/consistency_migrations/project/behaves_like_a_consistency_migration/record_is_consistent/should_update_is_consistent_to_true.yml diff --git a/spec/cassettes/db_data_migrate/_invoke/consistency_migrations/project/behaves_like_a_consistency_migration/record_is_consistent/should_update_is_consistent_to_true.yml b/spec/cassettes/db_data_migrate/_invoke/consistency_migrations/project/behaves_like_a_consistency_migration/record_is_consistent/should_update_is_consistent_to_true.yml deleted file mode 100644 index 0d98180f8..000000000 --- a/spec/cassettes/db_data_migrate/_invoke/consistency_migrations/project/behaves_like_a_consistency_migration/record_is_consistent/should_update_is_consistent_to_true.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -http_interactions: -- request: - method: get - uri: http://swift.local:12345/auth/v1.0 - body: - encoding: US-ASCII - string: '' - headers: - X-Auth-User: - - test:tester - X-Auth-Key: - - testing - response: - status: - code: 200 - message: OK - headers: - X-Storage-Url: - - http://swift.local:12345/v1/AUTH_test - X-Auth-Token: - - AUTH_tk65cd2a8981b5430c8f241c264a55a0de - Content-Type: - - text/html; charset=UTF-8 - X-Storage-Token: - - AUTH_tk65cd2a8981b5430c8f241c264a55a0de - Content-Length: - - '0' - X-Trans-Id: - - txd2948474bdc54223870ae-0059554e83 - Date: - - Thu, 29 Jun 2017 19:01:23 GMT - body: - encoding: UTF-8 - string: '' - http_version: - recorded_at: Thu, 29 Jun 2017 19:01:23 GMT -- request: - method: put - uri: http://swift.local:12345/v1/AUTH_test/54396c88-76ae-45a7-a8e8-a852f8931f5b - body: - encoding: UTF-8 - string: '' - headers: - X-Auth-Token: - - AUTH_tk65cd2a8981b5430c8f241c264a55a0de - X-Container-Meta-Access-Control-Allow-Origin: - - "*" - response: - status: - code: 201 - message: Created - headers: - Content-Length: - - '0' - Content-Type: - - text/html; charset=UTF-8 - X-Trans-Id: - - tx3276247faf4248d6b96e4-0059554e83 - Date: - - Thu, 29 Jun 2017 19:01:24 GMT - body: - encoding: UTF-8 - string: '' - http_version: - recorded_at: Thu, 29 Jun 2017 19:01:24 GMT -recorded_with: VCR 3.0.3 From b497ef2b91bff02e282a4372b8c2699cfa01099c Mon Sep 17 00:00:00 2001 From: Darrin Mann Date: Wed, 19 Jul 2017 15:34:20 -0400 Subject: [PATCH 3/3] Print more information during consistency_status db data migration to prevent timeouts --- lib/tasks/db_data_migrate.rake | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/lib/tasks/db_data_migrate.rake b/lib/tasks/db_data_migrate.rake index 17cb53bd5..9336de0d0 100644 --- a/lib/tasks/db_data_migrate.rake +++ b/lib/tasks/db_data_migrate.rake @@ -79,18 +79,22 @@ def migrate_nil_consistency_status storage_provider = StorageProvider.first updated_projects = 0 updated_uploads = 0 - Project.where(is_consistent: nil).each do |p| - unless p.is_deleted - if storage_provider.get_container_meta(p.id) - p.update_columns(is_consistent: true) - else - p.update_columns(is_consistent: false) - end - updated_projects += 1 + projects = Project.where(is_consistent: nil).where.not(is_deleted: true) + puts "#{projects.count} projects with nil consistency_status." + projects.each do |p| + if storage_provider.get_container_meta(p.id) + p.update_columns(is_consistent: true) + else + p.update_columns(is_consistent: false) end + print '.' + updated_projects += 1 end + puts "#{updated_projects} projects updated." - Upload.where(is_consistent: nil).each do |u| + uploads = Upload.where(is_consistent: nil) + puts "#{uploads.count} uploads with nil consistency_status." + uploads.each do |u| begin if storage_provider.get_object_metadata(u.project.id, u.id) u.update_columns(is_consistent: true) @@ -99,8 +103,9 @@ def migrate_nil_consistency_status u.update_columns(is_consistent: false) end updated_uploads += 1 + print '.' end - $stderr.print "#{updated_projects} projects and #{updated_uploads} uploads updated consistency" + puts "#{updated_uploads} uploads updated." end namespace :db do