-
Notifications
You must be signed in to change notification settings - Fork 2.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Migration: Upload migration files from profile settings and start processing #8274
base: develop
Are you sure you want to change the base?
Changes from 11 commits
9fb0909
c6c19b7
c539d8c
7fa8057
c82f6c3
26995bd
1eec8a4
5255767
901e3e9
6927c21
14743e1
2e7dbb5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -8,3 +8,4 @@ | |
//= link error_pages.css | ||
//= link admin.css | ||
//= link rtl.css | ||
//= link archive-uploader |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,24 @@ | ||
const checkProfileUploadButton = function() { | ||
let photoFiles = $("#profile-file-btn")[0].files; | ||
let profileFiles = $("#photos-file-btn")[0].files; | ||
if ((photoFiles.size + profileFiles.size) === 0) { | ||
$("#upload_profile_files").attr("disabled", "disabled"); | ||
} else { | ||
$("#upload_profile_files").removeAttr("disabled"); | ||
} | ||
}; | ||
const profileFileButton = document.getElementById("profile-file-btn"); | ||
const profileFileChosen = document.getElementById("profile-file-chosen"); | ||
|
||
const photosFileButton = document.getElementById("photos-file-btn"); | ||
const photosFileChosen = document.getElementById("photos-file-chosen"); | ||
|
||
profileFileButton.addEventListener("change", function() { | ||
profileFileChosen.textContent = this.files[0].name; | ||
checkProfileUploadButton(); | ||
}); | ||
|
||
photosFileButton.addEventListener("change", function() { | ||
photosFileChosen.textContent = this.files[0].name; | ||
checkProfileUploadButton(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 'checkProfileUploadButton' was used before it was defined |
||
}); |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -16,6 +16,20 @@ textarea { | |
box-shadow: none; | ||
} | ||
} | ||
|
||
.upload-btn-wrapper { | ||
display: inline-block; | ||
overflow: hidden; | ||
position: relative; | ||
|
||
input[type=file] { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Avoid qualifying attribute selectors with an element. |
||
font-size: 100px; | ||
left: 0; | ||
opacity: 0; | ||
position: absolute; | ||
top: 0; | ||
} | ||
} | ||
// scss-lint:enable QualifyingElement | ||
|
||
textarea { | ||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -122,6 +122,10 @@ def confirm_email | |||||
redirect_to edit_user_path | ||||||
end | ||||||
|
||||||
def import_parameter?(import_parameters) | ||||||
import_parameters[:profile_path] || import_parameters[:photos_path] | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do we support importing just photos without the profile? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. A user could first import his/her account and in another step the photos. These files are huge, that to separate them still makes sense. |
||||||
end | ||||||
|
||||||
private | ||||||
|
||||||
def user_params | ||||||
|
@@ -229,15 +233,31 @@ def change_email(user_data) | |||||
|
||||||
def upload_export_files(user_data) | ||||||
logger.info "Start importing account" | ||||||
@user.export = user_data[:export] if user_data[:export] | ||||||
@user.exported_photos_file = user_data[:exported_photos_file] if user_data[:exported_photos_file] | ||||||
if @user.save | ||||||
flash.now[:notice] = "Your account migration has been scheduled" | ||||||
|
||||||
import_parameters = copy_import_files(user_data) | ||||||
|
||||||
if import_parameter?(import_parameters) | ||||||
flash.now[:notice] = t("users.import.import_has_been_scheduled") | ||||||
else | ||||||
flash.now[:error] = "Your account migration could not be scheduled for the following reason:"\ | ||||||
" #{@user.errors.full_messages}" | ||||||
flash.now[:error] = t("users.import.import_has_no_files_received") | ||||||
end | ||||||
Workers::ImportUser.perform_async(@user.id) | ||||||
Workers::ImportUser.perform_async(@user.id, import_parameters) | ||||||
end | ||||||
|
||||||
def copy_import_files(user_data) | ||||||
{ | ||||||
profile_path: copy_import_file(user_data[:export]), | ||||||
photos_path: copy_import_file(user_data[:exported_photos_file]) | ||||||
} | ||||||
end | ||||||
|
||||||
def copy_import_file(tmp_file) | ||||||
return if tmp_file.empty? | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think you need to use
Suggested change
|
||||||
|
||||||
file_path_to_save_to = Rails.root.join("public", "uploads", "users", | ||||||
"#{current_user.username}_#{tmp_file.original_filename}") | ||||||
FileUtils.cp tmp_file.path, file_path_to_save_to | ||||||
file_path_to_save_to | ||||||
end | ||||||
|
||||||
def change_settings(user_data, successful="users.update.settings_updated", error="users.update.settings_not_updated") | ||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,24 +3,27 @@ | |
class ImportService | ||
include Diaspora::Logging | ||
|
||
def import_by_user(user, opts={}) | ||
import_by_files(user.export.current_path, user.exported_photos_file.current_path, user.username, opts) | ||
def import_by_user(user_name, import_parameters) | ||
profile_path = import_parameters["profile_path"] | ||
photos_path = import_parameters["photos_path"] | ||
|
||
import_by_files(profile_path, photos_path, user_name) | ||
end | ||
|
||
def import_by_files(path_to_profile, path_to_photos, username, opts={}) | ||
user = User.find_by(username: username) | ||
raise ArgumentError, "Username #{username} should exist before uploading photos." if user.nil? | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think raising exception from the sidekiq job would schedule this job to retry. If this error is irrecoverable we should instead finish the job without errors, possibly logging some error messages. |
||
|
||
if path_to_profile.present? | ||
logger.info "Import for profile #{username} at path #{path_to_profile} requested" | ||
import_user_profile(path_to_profile, username, opts.merge(photo_migration: path_to_photos.present?)) | ||
end | ||
|
||
user = User.find_by(username: username) | ||
raise ArgumentError, "Username #{username} should exist before uploading photos." if user.nil? | ||
|
||
if path_to_photos.present? | ||
logger.info("Importing photos from import file for '#{username}' from #{path_to_photos}") | ||
import_user_photos(user, path_to_photos) | ||
end | ||
remove_file_references(user) | ||
remove_import_files(path_to_profile, path_to_photos) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Cleaning up invalid data -- if the archive is in wrong format. If we don't clean up, then someone could abuse the pod by uploading invalid archives indefinitely. I guess we could handle at least certain known errors when the archive import fails, and clean up the files. But not in every case, because in some cases we may want to retry the import. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What do you mean? The 'import_user_profile' and 'import_user_photos' handle (known) errors. Whenever execution comes here, the uploaded files (I guess the zipped ones) will be removed. |
||
end | ||
|
||
private | ||
|
@@ -101,9 +104,11 @@ def create_folder(compressed_file_name) | |
folder | ||
end | ||
|
||
def remove_file_references(user) | ||
user.remove_exported_photos_file | ||
user.remove_export | ||
user.save | ||
# Removes import files after processing | ||
# @param [*String] files | ||
def remove_import_files(*files) | ||
files.each do |file| | ||
File.delete(file) if file && File.exist?(file) | ||
end | ||
end | ||
end |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -193,8 +193,11 @@ | |
%hr | ||
|
||
.row | ||
.col-md-6.account-data | ||
.col-md-12 | ||
%h3= t(".export_data") | ||
%p= t(".export_data_subline") | ||
.row | ||
.col-md-6.account-data | ||
%h4= t("profile") | ||
.form-group | ||
- if current_user.exporting | ||
|
@@ -210,6 +213,7 @@ | |
= link_to t(".request_export"), export_profile_user_path, method: :post, | ||
class: "btn btn-default" | ||
|
||
.col-md-6 | ||
%h4= t("javascripts.profile.photos") | ||
.form-group | ||
- if current_user.exporting_photos | ||
|
@@ -224,7 +228,20 @@ | |
= link_to t(".request_export_photos"), export_photos_user_path, method: :post, | ||
class: "btn btn-default" | ||
|
||
.col-md-6 | ||
.row | ||
.col-md-12 | ||
%h3 | ||
= t(".import.headline", {accountname: current_user.diaspora_handle}) | ||
%p | ||
= t(".import.body") | ||
%strong | ||
= t(".import.warning") | ||
.form-group | ||
.btn.btn-primary{id: "import_account", data: {toggle: "modal", target: "#importAccountModal"}} | ||
Import another account... | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this needs to be translatable -- wrap into |
||
= render "import_account_modal" | ||
.row | ||
.col-md-12 | ||
%h3 | ||
= t(".close_account_text") | ||
.form-group | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
.modal.fade{id: "importAccountModal", | ||
tabindex: "-1", | ||
role: "dialog", | ||
aria: {labelledby: "importAccountModalLabel", hidden: "true"}} | ||
.modal-dialog | ||
.modal-content | ||
.modal-header | ||
%button.close{type: "button", data: {dismiss: "modal"}, aria: {hidden: "true"}} | ||
× | ||
%h3.modal-title{id: "importAccountModalLabel"} | ||
= t("users.import_modal.title") | ||
.modal-body | ||
%p | ||
= simple_format(t("users.import_modal.subtext", {accountname: current_user.diaspora_handle})) | ||
%ol | ||
%li | ||
= t("users.import_modal.list1") | ||
%li | ||
= t("users.import_modal.list2") | ||
%li | ||
= t("users.import_modal.list3") | ||
%li | ||
= t("users.import_modal.list4") | ||
%strong | ||
= t("users.import_modal.closing_text") | ||
%p | ||
= t("users.import_modal.closing_recommendation") | ||
%p | ||
= t("users.import_modal.old_pod_offline") | ||
|
||
= form_for current_user, | ||
url: edit_user_path, | ||
html: {method: :put, multipart: true, class: "form-horizontal"} do |f| | ||
.row | ||
.col-md-12 | ||
.upload-btn-wrapper | ||
.btn.btn-primary | ||
= t("users.import_modal.select_profile_archive") | ||
= f.file_field :export, accept: "application/json, application/zip, application/gzip", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Style/StringLiterals: Prefer double-quoted strings unless you need single quotes to avoid extra backslashes for escaping. |
||
id: "profile-file-btn" | ||
%span#profile-file-chosen | ||
= t("users.import_modal.no_profile_file_set") | ||
.row | ||
.col-md-12 | ||
.upload-btn-wrapper | ||
.btn.btn-primary | ||
= t("users.import_modal.select_photo_archive") | ||
= f.file_field :exported_photos_file, accept: "application/zip", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
id: "photos-file-btn" | ||
%span#photos-file-chosen | ||
= t("users.import_modal.no_photos_file_set") | ||
%h4 | ||
= t("users.import_modal.import_starts") | ||
%strong | ||
= t("users.import_modal.original_account_will_be_deleted") | ||
%p | ||
.clearfix | ||
.btn.btn-default{data: {dismiss: "modal"}} | ||
= t("users.import_modal.cancel") | ||
= f.submit t("users.import_modal.start_import"), | ||
class: "btn btn-primary.pull-right", | ||
id: "upload_profile_files", | ||
disabled: true | ||
|
||
= javascript_include_tag "archive-uploader" |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,9 +4,9 @@ module Workers | |
class ImportUser < ArchiveBase | ||
private | ||
|
||
def perform_archive_job(user_id) | ||
def perform_archive_job(user_id, import_parameters) | ||
user = User.find(user_id) | ||
ImportService.new.import_by_user(user) | ||
ImportService.new.import_by_user(user.username, import_parameters) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. So we have a user object, and then we get it's Instead of changing this signature to username, maybe we could change |
||
end | ||
end | ||
end |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,7 @@ def substitute_author | |
|
||
entity_data["photos"].each do |photo| | ||
photo["entity_data"]["author"] = user.diaspora_handle | ||
photo["entity_data"]["remote_photo_path"] = "#{AppConfig.pod_uri}uploads\/images\/" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Okay, so this is how we link uploaded photos. If there are files in the photos archive that weren't in the end linked to any photo record, should we detect and clean them up? |
||
end | ||
end | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
'checkProfileUploadButton' was used before it was defined