Skip to content

Commit

Permalink
Merge 28062b4 into 2c74f33
Browse files Browse the repository at this point in the history
  • Loading branch information
tallenaz committed Jul 17, 2018
2 parents 2c74f33 + 28062b4 commit 291574a
Show file tree
Hide file tree
Showing 11 changed files with 45 additions and 38 deletions.
2 changes: 1 addition & 1 deletion app/jobs/plexer_job.rb
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def apcs
end

def find_or_create_unreplicated_part(apc, part_s3_key, metadata)
apc.archive_preserved_copy_parts.find_or_create_by(
apc.zip_parts.find_or_create_by(
create_info: metadata.slice(:zip_cmd, :zip_version).to_s,
md5: metadata[:checksum_md5],
parts_count: metadata[:parts_count],
Expand Down
8 changes: 4 additions & 4 deletions app/jobs/results_recorder_job.rb
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Preconditions:
# PlexerJob has made a matching ArchivePreservedCopyPart row
# PlexerJob has made a matching ZipPart row
#
# Responsibilities:
# Update DB per event info.
Expand All @@ -23,7 +23,7 @@ class ResultsRecorderJob < ApplicationJob
# @param [String] s3_part_key
# @param [String] delivery_class Name of the worker class that performed delivery
def perform(druid, version, s3_part_key, _delivery_class)
part = apc_part!(s3_part_key)
part = zip_part!(s3_part_key)
part.ok!
apc.ok! if part.all_parts_replicated? # are all of the parts replicated for this zip_endpoint?
# only publish result if all of the parts replicated for all zip_endpoints
Expand All @@ -33,9 +33,9 @@ def perform(druid, version, s3_part_key, _delivery_class)

private

def apc_part!(s3_part_key)
def zip_part!(s3_part_key)
raise "Status shifted underneath replication: #{apc.inspect}" unless apc.unreplicated?
apc.archive_preserved_copy_parts.find_by!(
apc.zip_parts.find_by!(
suffix: File.extname(s3_part_key),
status: 'unreplicated'
)
Expand Down
4 changes: 3 additions & 1 deletion app/models/archive_preserved_copy.rb
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

# Corresponds to a Moab-Version on a ZipEndpoint.
# There will be individual parts (at least one) - see ArchivepreservedCopyPart.
# For a fully consistent system, given an (Online) PreservedCopy, the number of associated
Expand All @@ -8,8 +9,9 @@
class ArchivePreservedCopy < ApplicationRecord
belongs_to :preserved_copy
belongs_to :zip_endpoint
has_many :archive_preserved_copy_parts, dependent: :destroy, inverse_of: :archive_preserved_copy
has_many :zip_parts, dependent: :destroy, inverse_of: :archive_preserved_copy
has_one :preserved_object, through: :preserved_copy, dependent: :restrict_with_exception

delegate :preserved_object, to: :preserved_copy

# @note Hash values cannot be modified without migrating any associated persisted data.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# We chunk archives of Moab versions into multiple files, so we don't get
# completely unwieldy file sizes. This represents metadata for one such part.
# This model's data is populated by PlexerJob.
class ArchivePreservedCopyPart < ApplicationRecord
belongs_to :archive_preserved_copy, inverse_of: :archive_preserved_copy_parts
class ZipPart < ApplicationRecord
belongs_to :archive_preserved_copy, inverse_of: :zip_parts
delegate :zip_endpoint, :preserved_copy, to: :archive_preserved_copy
delegate :preserved_object, to: :preserved_copy

Expand All @@ -17,11 +17,11 @@ class ArchivePreservedCopyPart < ApplicationRecord
validates :suffix, presence: true, format: { with: /\A\.z(ip|[0-9]+)\z/ }
validates :parts_count, presence: true, numericality: { only_integer: true, greater_than: 0 }

# For this persisted part, are it and all its cohort now replicated (to one zip_endpoint)?
# For this persisted part, are it and all its cohort now replicated (to one endpoint)?
# @return [Boolean] true if all expected parts are now replicated
def all_parts_replicated?
return false unless persisted? && ok?
parts = archive_preserved_copy.archive_preserved_copy_parts.where(suffix: suffixes_in_set)
parts = archive_preserved_copy.zip_parts.where(suffix: suffixes_in_set)
parts.count == parts_count && parts.all?(&:ok?)
end

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
class ChangeArchivePreservedCopyPartsToZipParts < ActiveRecord::Migration[5.1]
def change
rename_table :archive_preserved_copy_parts, :zip_parts
end
end
28 changes: 14 additions & 14 deletions db/schema.rb
Original file line number Diff line number Diff line change
Expand Up @@ -29,19 +29,6 @@
t.index ["zip_endpoint_id"], name: "index_archive_preserved_copies_on_zip_endpoint_id"
end

create_table "archive_preserved_copy_parts", force: :cascade do |t|
t.bigint "size"
t.bigint "archive_preserved_copy_id", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "md5", null: false
t.string "create_info", null: false
t.integer "parts_count", null: false
t.string "suffix", null: false
t.integer "status", default: 1, null: false
t.index ["archive_preserved_copy_id"], name: "index_archive_preserved_copy_parts_on_archive_preserved_copy_id"
end

create_table "endpoints", force: :cascade do |t|
t.string "endpoint_name", null: false
t.datetime "created_at", null: false
Expand Down Expand Up @@ -118,14 +105,27 @@
t.index ["endpoint_name"], name: "index_zip_endpoints_on_endpoint_name", unique: true
end

create_table "zip_parts", force: :cascade do |t|
t.bigint "size"
t.bigint "archive_preserved_copy_id", null: false
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.string "md5", null: false
t.string "create_info", null: false
t.integer "parts_count", null: false
t.string "suffix", null: false
t.integer "status", default: 1, null: false
t.index ["archive_preserved_copy_id"], name: "index_zip_parts_on_archive_preserved_copy_id"
end

add_foreign_key "archive_preserved_copies", "preserved_copies"
add_foreign_key "archive_preserved_copies", "zip_endpoints"
add_foreign_key "archive_preserved_copy_parts", "archive_preserved_copies"
add_foreign_key "endpoints_preservation_policies", "endpoints"
add_foreign_key "endpoints_preservation_policies", "preservation_policies"
add_foreign_key "preservation_policies_zip_endpoints", "preservation_policies"
add_foreign_key "preservation_policies_zip_endpoints", "zip_endpoints"
add_foreign_key "preserved_copies", "endpoints"
add_foreign_key "preserved_copies", "preserved_objects"
add_foreign_key "preserved_objects", "preservation_policies"
add_foreign_key "zip_parts", "archive_preserved_copies"
end
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
FactoryBot.define do
factory :archive_preserved_copy_part do
factory :zip_part do
md5 "00236a2ae558018ed13b5222ef1bd977"
create_info "ok"
parts_count 1
Expand Down
18 changes: 9 additions & 9 deletions spec/jobs/plexer_job_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -45,19 +45,19 @@
job.perform(druid, version, s3_key, metadata)
end

it 'ensures archive_preserved_copy_part exists with status unreplicated before queueing for delivery' do
it 'ensures zip_part exists with status unreplicated before queueing for delivery' do
skip('need test for ensuring part exists with status unreplicated')
end

it 'adds ArchivePreservedCopyPart to each related APC' do
it 'adds ZipPart to each related APC' do
job.perform(druid, version, s3_key, metadata)
apc1.archive_preserved_copy_parts.reload
apc2.archive_preserved_copy_parts.reload
expect(apc1.archive_preserved_copy_parts.count).to eq 1
expect(apc2.archive_preserved_copy_parts.count).to eq 1
expect(apc1.archive_preserved_copy_parts.first!.md5).to eq md5
expect(apc2.archive_preserved_copy_parts.first!.md5).to eq md5
expect(apc1.archive_preserved_copy_parts.first!.create_info).to eq metadata.slice(:zip_cmd, :zip_version).to_s
apc1.zip_parts.reload
apc2.zip_parts.reload
expect(apc1.zip_parts.count).to eq 1
expect(apc2.zip_parts.count).to eq 1
expect(apc1.zip_parts.first!.md5).to eq md5
expect(apc2.zip_parts.first!.md5).to eq md5
expect(apc1.zip_parts.first!.create_info).to eq metadata.slice(:zip_cmd, :zip_version).to_s
end
end
end
2 changes: 1 addition & 1 deletion spec/jobs/results_recorder_job_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
let(:druid) { apc.preserved_object.druid }
let(:zip_endpoint) { apc.zip_endpoint }

before { apc.archive_preserved_copy_parts.create(attributes_for(:archive_preserved_copy_part)) }
before { apc.zip_parts.create(attributes_for(:zip_part)) }

it 'descends from ApplicationJob' do
expect(described_class.new).to be_an(ApplicationJob)
Expand Down
2 changes: 1 addition & 1 deletion spec/models/archive_preserved_copy_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@

it { is_expected.to belong_to(:preserved_copy) }
it { is_expected.to belong_to(:zip_endpoint) }
it { is_expected.to have_many(:archive_preserved_copy_parts) }
it { is_expected.to have_db_index(:zip_endpoint_id) }
it { is_expected.to have_many(:zip_parts) }
it { is_expected.to have_db_index(:last_existence_check) }
it { is_expected.to have_db_index(:preserved_copy_id) }
it { is_expected.to have_db_index(:status) }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
require 'rails_helper'

RSpec.describe ArchivePreservedCopyPart, type: :model do
RSpec.describe ZipPart, type: :model do
let(:apc) { create(:archive_preserved_copy) }
let(:args) { attributes_for(:archive_preserved_copy_part).merge(archive_preserved_copy: apc) }
let(:args) { attributes_for(:zip_part).merge(archive_preserved_copy: apc) }

it 'defines a status enum with the expected values' do
is_expected.to define_enum_for(:status).with(
Expand Down

0 comments on commit 291574a

Please sign in to comment.