Permalink
Browse files

FIX: Add compatibility for bucket folder paths in migrate_to_s3 task (#…

…6855)

* FIX: Add compatibility for bucket folder paths in migrate_to_s3 task
* Refactor bucket_name split logic into S3Helper
  • Loading branch information...
rishabhnambiar committed Jan 8, 2019
1 parent 733a60e commit f181e9cc08a5d3c31598587957f8999b7e62ac50
Showing with 24 additions and 10 deletions.
  1. +7 −3 lib/s3_helper.rb
  2. +17 −7 lib/tasks/uploads.rake
@@ -12,7 +12,7 @@ def initialize(s3_bucket_name, tombstone_prefix = '', options = {})

@s3_bucket_name, @s3_bucket_folder_path = begin
raise Discourse::InvalidParameters.new("s3_bucket_name") if s3_bucket_name.blank?
s3_bucket_name.downcase.split("/".freeze, 2)
self.class.get_bucket_and_folder_path(s3_bucket_name)
end

@tombstone_prefix =
@@ -23,6 +23,10 @@ def initialize(s3_bucket_name, tombstone_prefix = '', options = {})
end
end

def self.get_bucket_and_folder_path(s3_bucket_name)
s3_bucket_name.downcase.split("/".freeze, 2)
end

def upload(file, path, options = {})
path = get_path_for_s3_upload(path)
obj = s3_bucket.object(path)
@@ -62,10 +66,10 @@ def copy(source, destination, options: {})
options[:copy_source] = File.join(@s3_bucket_name, source)
else
if @s3_bucket_folder_path
bucket_folder, filename = begin
folder, filename = begin
source.split("/".freeze, 2)
end
options[:copy_source] = File.join(@s3_bucket_name, bucket_folder, multisite_upload_path, filename)
options[:copy_source] = File.join(@s3_bucket_name, folder, multisite_upload_path, filename)
else
options[:copy_source] = File.join(@s3_bucket_name, multisite_upload_path, source)
end
@@ -214,6 +214,7 @@ def migrate_to_s3
db = RailsMultisite::ConnectionManagement.current_db

dry_run = !!ENV["DRY_RUN"]
bucket_has_folder_path = true if ENV["DISCOURSE_S3_BUCKET"].include? "/"

puts "*" * 30 + " DRY RUN " + "*" * 30 if dry_run
puts "Migrating uploads to S3 for '#{db}'..."
@@ -246,11 +247,18 @@ def migrate_to_s3

s3 = Aws::S3::Client.new(S3Helper.s3_options(GlobalSetting))

if bucket_has_folder_path
bucket, folder = S3Helper.get_bucket_and_folder_path(ENV["DISCOURSE_S3_BUCKET"])
folder = File.join(folder, "/")
else
bucket, folder = GlobalSetting.s3_bucket, ""
end

begin
s3.head_bucket(bucket: GlobalSetting.s3_bucket)
s3.head_bucket(bucket: bucket)
rescue Aws::S3::Errors::NotFound
puts "Bucket '#{GlobalSetting.s3_bucket}' not found. Creating it..."
s3.create_bucket(bucket: GlobalSetting.s3_bucket) unless dry_run
puts "Bucket '#{bucket}' not found. Creating it..."
s3.create_bucket(bucket: bucket) unless dry_run
end

puts "Uploading files to S3..."
@@ -267,7 +275,7 @@ def migrate_to_s3

s3_objects = []
prefix = Rails.configuration.multisite ? "#{db}/original/" : "original/"
options = { bucket: GlobalSetting.s3_bucket, prefix: prefix }
options = { bucket: bucket, prefix: prefix }

loop do
response = s3.list_objects_v2(options)
@@ -287,6 +295,8 @@ def migrate_to_s3
path = File.join("public", file)
name = File.basename(path)
etag = Digest::MD5.file(path).hexdigest
key = file[file.index(prefix)..-1]
key.prepend(folder) if bucket_has_folder_path

if s3_object = s3_objects.find { |obj| file.ends_with?(obj.key) }
next if File.size(path) == s3_object.size && s3_object.etag[etag]
@@ -295,9 +305,9 @@ def migrate_to_s3
options = {
acl: "public-read",
body: File.open(path, "rb"),
bucket: GlobalSetting.s3_bucket,
bucket: bucket,
content_type: MiniMime.lookup_by_filename(name)&.content_type,
key: file[file.index(prefix)..-1],
key: key,
}

if !FileHelper.is_supported_image?(name)
@@ -337,7 +347,7 @@ def migrate_to_s3
}

from = "/uploads/#{db}/original/(\\dX/(?:[a-f0-9]/)*[a-f0-9]{40}[a-z0-9\\.]*)"
to = "#{SiteSetting.Upload.s3_base_url}/#{prefix}\\1"
to = "#{SiteSetting.Upload.s3_base_url}/#{folder}#{prefix}\\1"

if dry_run
puts "REPLACING '#{from}' WITH '#{to}'"

1 comment on commit f181e9c

@discoursebot

This comment has been minimized.

Copy link

discoursebot commented on f181e9c Jan 11, 2019

This commit has been mentioned on Discourse Meta. There might be relevant details there:

https://meta.discourse.org/t/s3-migrations-from-to-minio-problems/97333/39

Please sign in to comment.