From 61de397815b74ff2b05d2bc50f14f5cb1c932865 Mon Sep 17 00:00:00 2001 From: Hector Correa Date: Mon, 3 Jun 2024 14:41:55 -0400 Subject: [PATCH] Updated the tests to account for the new files and directory Co-authored-by: Robert-Anthony Lee-Faison --- spec/services/s3_query_service_spec.rb | 92 +++++++++++++++----------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/spec/services/s3_query_service_spec.rb b/spec/services/s3_query_service_spec.rb index 043094d1..b05b5db2 100644 --- a/spec/services/s3_query_service_spec.rb +++ b/spec/services/s3_query_service_spec.rb @@ -9,10 +9,13 @@ let(:s3_bucket_key) { "10.34770/pe9w-x904/#{work.id}/" } let(:s3_key1) { "#{s3_bucket_key}SCoData_combined_v1_2020-07_README.txt" } let(:s3_key2) { "#{s3_bucket_key}SCoData_combined_v1_2020-07_datapackage.json" } + let(:s3_key3) { "#{s3_bucket_key}a_directory/" } let(:s3_last_modified1) { Time.parse("2022-04-21T18:29:40.000Z") } let(:s3_last_modified2) { Time.parse("2022-04-21T18:30:07.000Z") } + let(:s3_last_modified3) { Time.parse("2022-05-21T18:31:07.000Z") } let(:s3_size1) { 5_368_709_122 } let(:s3_size2) { 5_368_709_128 } + let(:s3_size3) { 0 } let(:s3_etag1) { "008eec11c39e7038409739c0160a793a" } let(:s3_hash) do { @@ -34,9 +37,9 @@ }, { etag: "\"7bd3d4339c034ebc663b99065771111\"", - key: "a_directory/", - last_modified: s3_last_modified2, - size: 0, + key: s3_key3, + last_modified: s3_last_modified3, + size: s3_size3, storage_class: "STANDARD" } ], @@ -119,7 +122,7 @@ data_profile = s3_query_service.data_profile expect(data_profile[:objects]).to be_instance_of(Array) expect(data_profile[:ok]).to eq true - expect(data_profile[:objects].count).to eq 2 + expect(data_profile[:objects].count).to eq 3 expect(data_profile[:objects].first).to be_instance_of(S3File) expect(data_profile[:objects].first.filename).to match(/README/) expect(data_profile[:objects].first.last_modified).to eq Time.parse("2022-04-21T18:29:40.000Z") @@ -137,11 +140,13 @@ data_profile = s3_query_service.data_profile expect(data_profile[:objects]).to be_instance_of(Array) expect(data_profile[:ok]).to eq true - expect(data_profile[:objects].count).to eq 4 - expect(data_profile[:objects].first.filename).to match(/README/) + expect(data_profile[:objects].count).to eq 6 + expect(data_profile[:objects][0].filename).to match(/README/) expect(data_profile[:objects][1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(data_profile[:objects][2].filename).to match(/README/) - expect(data_profile[:objects][3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(data_profile[:objects][2].filename).to match(/a_directory/) + expect(data_profile[:objects][3].filename).to match(/README/) + expect(data_profile[:objects][4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(data_profile[:objects][5].filename).to match(/a_directory/) end it "handles connecting to a bad bucket" do @@ -214,7 +219,9 @@ { "filename" => "10.34770/pe9w-x904/#{work.id}/SCoData_combined_v1_2020-07_README.txt", "snapshot_id" => snapshot.id, "upload_status" => "started", "user_id" => user.id, "checksum" => "008eec11c39e7038409739c0160a793a" }, { "filename" => "10.34770/pe9w-x904/#{work.id}/SCoData_combined_v1_2020-07_datapackage.json", "snapshot_id" => snapshot.id, "upload_status" => "started", - "user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b990657714688" } + "user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b990657714688" }, + { "filename" => "10.34770/pe9w-x904/#{work.id}/a_directory/", "snapshot_id" => snapshot.id, "upload_status" => "started", + "user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b99065771111" } ]) perform_enqueued_jobs expect(s3_query_service.client).to have_received(:create_multipart_upload) @@ -326,30 +333,34 @@ expect(data_profile).to include(:objects) children = data_profile[:objects] - expect(children.count).to eq 2 + expect(children.count).to eq 3 + expect(children.first).to be_instance_of(S3File) expect(children.first.filename).to eq(s3_key1) - - last_modified = children.first.last_modified - expect(last_modified.to_s).to eq(s3_last_modified1.to_s) - + expect(children.first.last_modified.to_s).to eq(s3_last_modified1.to_s) expect(children.first.size).to eq(s3_size1) - expect(children.last).to be_instance_of(S3File) - expect(children.last.filename).to eq(s3_key2) + expect(children.second).to be_instance_of(S3File) + expect(children.second.filename).to eq(s3_key2) + expect(children.second.last_modified.to_s).to eq(s3_last_modified2.to_s) + expect(children.second.size).to eq(s3_size2) - last_modified = children.last.last_modified - expect(last_modified.to_s).to eq(s3_last_modified2.to_s) - - expect(children.last.size).to eq(s3_size2) + expect(children.last).to be_instance_of(S3File) + expect(children.last.filename).to eq(s3_key3) + expect(children.last.last_modified.to_s).to eq(s3_last_modified3.to_s) + expect(children.last.size).to eq(s3_size3) end end describe "#file_count" do - it "returns only the files" do - expect(s3_query_service.file_count).to eq(2) + it "returns files and directories" do + expect(s3_query_service.file_count).to eq(3) end + # = + # TODO: Add a test that checks for files only + # = + context "when an error is encountered" do subject(:s3_query_service) { described_class.new(work) } let(:file_count) { s3_query_service.file_count } @@ -422,7 +433,7 @@ data_profile = s3_query_service.data_profile expect(data_profile[:objects]).to be_instance_of(Array) expect(data_profile[:ok]).to eq true - expect(data_profile[:objects].count).to eq 2 + expect(data_profile[:objects].count).to eq 3 expect(data_profile[:objects].first).to be_instance_of(S3File) expect(data_profile[:objects].first.filename).to match(/README/) expect(data_profile[:objects].first.last_modified).to eq Time.parse("2022-04-21T18:29:40.000Z") @@ -705,11 +716,13 @@ it "it retrieves the files for the work" do files = s3_query_service.client_s3_files - expect(files.count).to eq 4 - expect(files.first.filename).to match(/README/) + expect(files.count).to eq 6 + expect(files[0].filename).to match(/README/) expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(files[2].filename).to match(/README/) - expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[2].filename).to match(/a_directory/) + expect(files[3].filename).to match(/README/) + expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[5].filename).to match(/a_directory/) expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "example-bucket", max_keys: 1000, prefix: "10.34770/pe9w-x904/#{work.id}/") expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "example-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "10.34770/pe9w-x904/#{work.id}/") end @@ -717,11 +730,13 @@ it "it retrieves the files for a bucket and prefix once" do s3_query_service.client_s3_files(bucket_name: "other-bucket", prefix: "new-prefix") files = s3_query_service.client_s3_files(bucket_name: "other-bucket", prefix: "new-prefix") - expect(files.count).to eq 4 - expect(files.first.filename).to match(/README/) + expect(files.count).to eq 6 + expect(files[0].filename).to match(/README/) expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(files[2].filename).to match(/README/) - expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[2].filename).to match(/a_directory/) + expect(files[3].filename).to match(/README/) + expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[5].filename).to match(/a_directory/) expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix").once expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix").once end @@ -729,11 +744,13 @@ it "it retrieves the files for a bucket and prefix any time reload is true" do s3_query_service.client_s3_files(reload: true, bucket_name: "other-bucket", prefix: "new-prefix") files = s3_query_service.client_s3_files(reload: true, bucket_name: "other-bucket", prefix: "new-prefix") - expect(files.count).to eq 4 - expect(files.first.filename).to match(/README/) + expect(files.count).to eq 6 + expect(files[0].filename).to match(/README/) expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(files[2].filename).to match(/README/) - expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[2].filename).to match(/a_directory/) + expect(files[3].filename).to match(/README/) + expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) + expect(files[5].filename).to match(/a_directory/) expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix").twice expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix").twice end @@ -743,10 +760,10 @@ expect(files.count).to eq 6 expect(files[0].filename).to match(/README/) expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(files[2].filename).to match(/directory/) + expect(files[2].filename).to match(/a_directory/) expect(files[3].filename).to match(/README/) expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/) - expect(files[5].filename).to match(/directory/) + expect(files[5].filename).to match(/a_directory/) expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix") expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix") end @@ -852,6 +869,7 @@ s3_query_service.publish_files(user) end + # TODO: Make sure this still works since it depends on the empty files describe "#publish_files" do it "creates a work activity identifying the empty files" do work.reload