Skip to content

Commit

Permalink
Updated the tests to account for the new files and directory
Browse files Browse the repository at this point in the history 
Co-authored-by: Robert-Anthony Lee-Faison <leefaisonr@users.noreply.github.com>
  • Loading branch information
@hectorcorrea @leefaisonr
hectorcorrea and leefaisonr committed Jun 3, 2024
1 parent cbca1f0 commit 61de397
Showing 1 changed file with 55 additions and 37 deletions.
92 changes: 55 additions & 37 deletions spec/services/s3_query_service_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,13 @@
let(:s3_bucket_key) { "10.34770/pe9w-x904/#{work.id}/" }
let(:s3_key1) { "#{s3_bucket_key}SCoData_combined_v1_2020-07_README.txt" }
let(:s3_key2) { "#{s3_bucket_key}SCoData_combined_v1_2020-07_datapackage.json" }
let(:s3_key3) { "#{s3_bucket_key}a_directory/" }
let(:s3_last_modified1) { Time.parse("2022-04-21T18:29:40.000Z") }
let(:s3_last_modified2) { Time.parse("2022-04-21T18:30:07.000Z") }
let(:s3_last_modified3) { Time.parse("2022-05-21T18:31:07.000Z") }
let(:s3_size1) { 5_368_709_122 }
let(:s3_size2) { 5_368_709_128 }
let(:s3_size3) { 0 }
let(:s3_etag1) { "008eec11c39e7038409739c0160a793a" }
let(:s3_hash) do
{
Expand All @@ -34,9 +37,9 @@
},
{
etag: "\"7bd3d4339c034ebc663b99065771111\"",
key: "a_directory/",
last_modified: s3_last_modified2,
size: 0,
key: s3_key3,
last_modified: s3_last_modified3,
size: s3_size3,
storage_class: "STANDARD"
}
],
Expand Down Expand Up @@ -119,7 +122,7 @@
data_profile = s3_query_service.data_profile
expect(data_profile[:objects]).to be_instance_of(Array)
expect(data_profile[:ok]).to eq true
expect(data_profile[:objects].count).to eq 2
expect(data_profile[:objects].count).to eq 3
expect(data_profile[:objects].first).to be_instance_of(S3File)
expect(data_profile[:objects].first.filename).to match(/README/)
expect(data_profile[:objects].first.last_modified).to eq Time.parse("2022-04-21T18:29:40.000Z")
Expand All @@ -137,11 +140,13 @@
data_profile = s3_query_service.data_profile
expect(data_profile[:objects]).to be_instance_of(Array)
expect(data_profile[:ok]).to eq true
expect(data_profile[:objects].count).to eq 4
expect(data_profile[:objects].first.filename).to match(/README/)
expect(data_profile[:objects].count).to eq 6
expect(data_profile[:objects][0].filename).to match(/README/)
expect(data_profile[:objects][1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(data_profile[:objects][2].filename).to match(/README/)
expect(data_profile[:objects][3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(data_profile[:objects][2].filename).to match(/a_directory/)
expect(data_profile[:objects][3].filename).to match(/README/)
expect(data_profile[:objects][4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(data_profile[:objects][5].filename).to match(/a_directory/)
end

it "handles connecting to a bad bucket" do
Expand Down Expand Up @@ -214,7 +219,9 @@
{ "filename" => "10.34770/pe9w-x904/#{work.id}/SCoData_combined_v1_2020-07_README.txt", "snapshot_id" => snapshot.id, "upload_status" => "started",
"user_id" => user.id, "checksum" => "008eec11c39e7038409739c0160a793a" },
{ "filename" => "10.34770/pe9w-x904/#{work.id}/SCoData_combined_v1_2020-07_datapackage.json", "snapshot_id" => snapshot.id, "upload_status" => "started",
"user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b990657714688" }
"user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b990657714688" },
{ "filename" => "10.34770/pe9w-x904/#{work.id}/a_directory/", "snapshot_id" => snapshot.id, "upload_status" => "started",
"user_id" => user.id, "checksum" => "7bd3d4339c034ebc663b99065771111" }
])
perform_enqueued_jobs
expect(s3_query_service.client).to have_received(:create_multipart_upload)
Expand Down Expand Up @@ -326,30 +333,34 @@
expect(data_profile).to include(:objects)
children = data_profile[:objects]

expect(children.count).to eq 2
expect(children.count).to eq 3

expect(children.first).to be_instance_of(S3File)
expect(children.first.filename).to eq(s3_key1)

last_modified = children.first.last_modified
expect(last_modified.to_s).to eq(s3_last_modified1.to_s)

expect(children.first.last_modified.to_s).to eq(s3_last_modified1.to_s)
expect(children.first.size).to eq(s3_size1)

expect(children.last).to be_instance_of(S3File)
expect(children.last.filename).to eq(s3_key2)
expect(children.second).to be_instance_of(S3File)
expect(children.second.filename).to eq(s3_key2)
expect(children.second.last_modified.to_s).to eq(s3_last_modified2.to_s)
expect(children.second.size).to eq(s3_size2)

last_modified = children.last.last_modified
expect(last_modified.to_s).to eq(s3_last_modified2.to_s)

expect(children.last.size).to eq(s3_size2)
expect(children.last).to be_instance_of(S3File)
expect(children.last.filename).to eq(s3_key3)
expect(children.last.last_modified.to_s).to eq(s3_last_modified3.to_s)
expect(children.last.size).to eq(s3_size3)
end
end

describe "#file_count" do
it "returns only the files" do
expect(s3_query_service.file_count).to eq(2)
it "returns files and directories" do
expect(s3_query_service.file_count).to eq(3)
end

# =
# TODO: Add a test that checks for files only
# =

context "when an error is encountered" do
subject(:s3_query_service) { described_class.new(work) }
let(:file_count) { s3_query_service.file_count }
Expand Down Expand Up @@ -422,7 +433,7 @@
data_profile = s3_query_service.data_profile
expect(data_profile[:objects]).to be_instance_of(Array)
expect(data_profile[:ok]).to eq true
expect(data_profile[:objects].count).to eq 2
expect(data_profile[:objects].count).to eq 3
expect(data_profile[:objects].first).to be_instance_of(S3File)
expect(data_profile[:objects].first.filename).to match(/README/)
expect(data_profile[:objects].first.last_modified).to eq Time.parse("2022-04-21T18:29:40.000Z")
Expand Down Expand Up @@ -705,35 +716,41 @@

it "it retrieves the files for the work" do
files = s3_query_service.client_s3_files
expect(files.count).to eq 4
expect(files.first.filename).to match(/README/)
expect(files.count).to eq 6
expect(files[0].filename).to match(/README/)
expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/README/)
expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/a_directory/)
expect(files[3].filename).to match(/README/)
expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[5].filename).to match(/a_directory/)
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "example-bucket", max_keys: 1000, prefix: "10.34770/pe9w-x904/#{work.id}/")
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "example-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "10.34770/pe9w-x904/#{work.id}/")
end

it "it retrieves the files for a bucket and prefix once" do
s3_query_service.client_s3_files(bucket_name: "other-bucket", prefix: "new-prefix")
files = s3_query_service.client_s3_files(bucket_name: "other-bucket", prefix: "new-prefix")
expect(files.count).to eq 4
expect(files.first.filename).to match(/README/)
expect(files.count).to eq 6
expect(files[0].filename).to match(/README/)
expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/README/)
expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/a_directory/)
expect(files[3].filename).to match(/README/)
expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[5].filename).to match(/a_directory/)
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix").once
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix").once
end

it "it retrieves the files for a bucket and prefix any time reload is true" do
s3_query_service.client_s3_files(reload: true, bucket_name: "other-bucket", prefix: "new-prefix")
files = s3_query_service.client_s3_files(reload: true, bucket_name: "other-bucket", prefix: "new-prefix")
expect(files.count).to eq 4
expect(files.first.filename).to match(/README/)
expect(files.count).to eq 6
expect(files[0].filename).to match(/README/)
expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/README/)
expect(files[3].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/a_directory/)
expect(files[3].filename).to match(/README/)
expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[5].filename).to match(/a_directory/)
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix").twice
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix").twice
end
Expand All @@ -743,10 +760,10 @@
expect(files.count).to eq 6
expect(files[0].filename).to match(/README/)
expect(files[1].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[2].filename).to match(/directory/)
expect(files[2].filename).to match(/a_directory/)
expect(files[3].filename).to match(/README/)
expect(files[4].filename).to match(/SCoData_combined_v1_2020-07_datapackage.json/)
expect(files[5].filename).to match(/directory/)
expect(files[5].filename).to match(/a_directory/)
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", max_keys: 1000, prefix: "new-prefix")
expect(fake_aws_client).to have_received(:list_objects_v2).with(bucket: "other-bucket", continuation_token: "abc123", max_keys: 1000, prefix: "new-prefix")
end
Expand Down Expand Up @@ -852,6 +869,7 @@
s3_query_service.publish_files(user)
end

# TODO: Make sure this still works since it depends on the empty files
describe "#publish_files" do
it "creates a work activity identifying the empty files" do
work.reload
Expand Down

0 comments on commit 61de397

Please sign in to comment.