-
-
Notifications
You must be signed in to change notification settings - Fork 195
/
storage.rb
201 lines (154 loc) · 4.93 KB
/
storage.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
##
# Helper class to:
# 1. migrate file into ActiveStorage
# 2. mirror ActiveStorage blobs between secondary mirrors
# 3. promote ActiveStorage secondary mirrors to serve blobs data
#
# Require a ActiveRecord class and symbol representing the has_one_attached
# association.
#
class Storage
def initialize(klass, association, setter: :data=, getter: :data,
condition: nil)
@klass = klass
@association = association
@setter = setter
@getter = getter
@condition = condition
end
def migrate
count = unattached_files.count
unattached_files.find_each.with_index do |file, index|
Kernel.silence_warnings do
do_migrate = true
do_migrate = @condition.call(file) if @condition
file.public_send(@setter, file.public_send(@getter)) if do_migrate
end
print "#{prefix}: Migrated #{index + 1}/#{count}"
rescue Errno::ENOENT
erase_line
Kernel.silence_warnings do
$stderr.puts "#{prefix};ID=#{file.id}: Missing #{file.filepath}."
end
end
puts "#{prefix}: Migrated old files storage to #{service_name} completed."
end
def mirror
return puts(not_a_mirror) unless mirror_service?
return puts(mirror_primary_not_disk_service) unless disk_service?
count = mirrorable_blobs.count
mirrorable_blobs.find_each.with_index do |blob, index|
begin
mirror_service.mirror(blob.key, checksum: blob.checksum)
rescue ActiveStorage::IntegrityError => ex
raise ex unless @klass == FoiAttachment
# Fix for https://github.com/mysociety/alaveteli/issues/8181
attachment = FoiAttachment.joins(:file_blob).
find_by(active_storage_blobs: { id: blob })
# Running the attachment masking will also mirror the file
FoiAttachmentMaskJob.set(queue: :low).perform_later(attachment)
end
print "#{prefix}: Mirrored #{index + 1}/#{count}"
end
puts "#{prefix}: Mirrored from #{disk_service.name} to " \
"#{secondary_service.name} completed."
end
def promote
return puts(not_a_mirror) unless mirror_service?
return puts(mirror_primary_not_disk_service) unless disk_service?
count = promotable_blobs.count
promotable_blobs.find_each.with_index do |blob, index|
next unless secondary_service.exist?(blob.key)
blob.update(service_name: secondary_service.name)
print "#{prefix}: Promote #{index + 1}/#{count}"
end
puts "#{prefix}: Promoted blobs in #{disk_service.name} to " \
"#{secondary_service.name} completed."
end
def unlink
return puts(not_a_mirror) unless mirror_service?
return puts(mirror_primary_not_disk_service) unless disk_service?
count = secondary_blobs.count
secondary_blobs.find_each.with_index do |blob, index|
next unless disk_service.exist?(blob.key)
disk_service.delete(blob.key)
print "#{prefix}: Unlink #{index + 1}/#{count}"
end
puts "#{prefix}: Unlinked files in #{disk_service.name} completed."
end
private
def unattached_files
@klass.left_joins(:"#{@association}_attachment").where(
active_storage_attachments: { id: nil }
)
end
def blobs
ActiveStorage::Blob.joins(:attachments).where(
active_storage_attachments: {
name: @association, record_type: @klass.to_s
}
)
end
def mirrorable_blobs
blobs.where(service_name: mirror_service.name)
end
def promotable_blobs
mirrorable_blobs.where(created_at: (..7.days.ago))
end
def secondary_blobs
blobs.where(service_name: secondary_service.name)
end
def attachment
@klass.reflect_on_attachment(@association)
end
def service_name
attachment.options[:service_name]
end
def service
ActiveStorage::Blob.services.fetch(service_name)
end
def mirror_service?
service.respond_to?(:mirror)
end
def mirror_service
raise not_a_mirror unless mirror_service?
service
end
def not_a_mirror
"#{prefix}: Not using the mirror service, ensure config/storage.yml is " \
"correct. See: " \
"https://alaveteli.org/docs/installing/storage/#mirrored-services"
end
def mirror_primary_not_disk_service
"#{prefix}: Mirror primary service is not a disk service, ensure " \
"config/storage.yml is correct. See: " \
"https://alaveteli.org/docs/installing/storage/#mirrored-services"
end
def disk_service
raise mirror_primary_not_disk_service unless disk_service?
mirror_service.primary
end
def disk_service?
mirror_service.primary.is_a?(ActiveStorage::Service::DiskService)
end
def secondary_service
mirror_service.mirrors.first
end
def prefix
@klass.to_s
end
def puts(*args)
return unless Rake.verbose
erase_line
$stdout.puts(*args)
end
def print(*args)
return unless Rake.verbose
erase_line
$stdout.print(*args)
end
def erase_line
# https://en.wikipedia.org/wiki/ANSI_escape_code#Escape_sequences
$stdout.print "\e[1G\e[K"
end
end