Upload migration files from profile settings and start processing

Author: tclaus

.gitignore

@@ -77,3 +77,4 @@ diaspora.iml
 
 # WebTranslateIt
 .wti
+/__MACOSX/

app/controllers/users_controller.rb

@@ -154,6 +154,8 @@ def user_params
       :post_default_public,
       :otp_required_for_login,
       :otp_secret,
+      :exported_photos_file,
+      :export,
       email_preferences: UserPreference::VALID_EMAIL_TYPES.map(&:to_sym)
     )
   end
@@ -172,6 +174,8 @@ def update_user(user_data)
       change_post_default(user_data)
     elsif user_data[:color_theme]
       change_settings(user_data, "users.update.color_theme_changed", "users.update.color_theme_not_changed")
+    elsif user_data[:export] || user_data[:exported_photos_file]
+      upload_export_files(user_data)
     else
       change_settings(user_data)
     end
@@ -235,6 +239,22 @@ def change_email(user_data)
     end
   end
 
+  def upload_export_files(user_data)
+    logger.info "Start importing profile"
+    @user.export = user_data[:export] if user_data[:export]
+    @user.exported_photos_file = user_data[:exported_photos_file] if user_data[:exported_photos_file]
+    if @user.save
+      flash.now[:notice] = "A profile migration is scheduled"
+    else
+      flash.now[:error] = "An error occured scheduling a migration: #{@user.errors.full_messages}"
+    end
+    start_migration_account
+  end
+
+  def start_migration_account
+    Workers::ImportProfile.perform_async(@user.username)
+  end
+
   def change_settings(user_data, successful="users.update.settings_updated", error="users.update.settings_not_updated")
     if @user.update_attributes(user_data)
       flash.now[:notice] = t(successful)

app/models/photo.rb

@@ -22,7 +22,7 @@ class Photo < ApplicationRecord
         large:  photo.url(:scaled_full),
         raw:    photo.url
       }
-    }, :as => :sizes
+    }, as: :sizes
     t.add lambda { |photo|
       {
         height: photo.height,
@@ -48,25 +48,25 @@ class Photo < ApplicationRecord
   before_destroy :ensure_user_picture
   after_destroy :clear_empty_status_message
 
-  after_commit :on => :create do
-    queue_processing_job if self.author.local?
+  after_commit on: :create do
+    queue_processing_job if author.local?
 
   end
 
   scope :on_statuses, ->(post_guids) {
-    where(:status_message_guid => post_guids)
+    where(status_message_guid: post_guids)
   }
 
   def clear_empty_status_message
-    if self.status_message && self.status_message.text_and_photos_blank?
-      self.status_message.destroy
+    if status_message&.text_and_photos_blank?
+      status_message.destroy
     else
       true
     end
   end
 
   def ownership_of_status_message
-    message = StatusMessage.find_by_guid(self.status_message_guid)
+    message = StatusMessage.find_by(guid: status_message_guid)
     return unless status_message_guid && message && diaspora_handle != message.diaspora_handle
 
     errors.add(:base, "Photo must have the same owner as status message")
@@ -96,21 +96,23 @@ def processed?
   end
 
   def update_remote_path
-    unless self.unprocessed_image.url.match(/^https?:\/\//)
-      remote_path = "#{AppConfig.pod_uri.to_s.chomp("/")}#{self.unprocessed_image.url}"
-    else
-      remote_path = self.unprocessed_image.url
-    end
+    remote_path = if unprocessed_image.url.match(%r{^https?://})
+                    unprocessed_image.url
+                  else
+                    "#{AppConfig.pod_uri.to_s.chomp('/')}#{unprocessed_image.url}"
+                  end
 
-    name_start = remote_path.rindex '/'
+    name_start = remote_path.rindex "/"
     self.remote_photo_path = "#{remote_path.slice(0, name_start)}/"
     self.remote_photo_name = remote_path.slice(name_start + 1, remote_path.length)
   end
 
-  def url(name = nil)
-    if remote_photo_path
-      name = name.to_s + '_' if name
-      image_url = remote_photo_path + name.to_s + remote_photo_name
+  def url(name=nil)
+    # During migration in bad cases this might be happen
+    # If this happens, stream loading stops. Better dont show photos as stop loading stream
+    if remote_photo_path.present? && remote_photo_name.present?
+      name = "#{name}_"
+      image_url = remote_photo_path + name + remote_photo_name
       if AppConfig.privacy.camo.proxy_remote_pod_images?
         Diaspora::Camo.image_url(image_url)
       else
@@ -124,15 +126,15 @@ def url(name = nil)
   end
 
   def ensure_user_picture
-    profiles = Profile.where(:image_url => url(:thumb_large))
+    profiles = Profile.where(image_url: url(:thumb_large))
     profiles.each { |profile|
       profile.image_url = nil
       profile.save
     }
   end
 
   def queue_processing_job
-    Workers::ProcessPhoto.perform_async(self.id)
+    Workers::ProcessPhoto.perform_async(id)
   end
 
   def self.visible(current_user, person, limit=:all, max_time=nil)

app/models/user.rb

@@ -312,6 +312,9 @@ def like_for(target)
   ######### Data export ##################
   mount_uploader :export, ExportedUser
 
+  ######### Photos export ##################
+  mount_uploader :exported_photos_file, ExportedPhotos
+
   def queue_export
     update exporting: true, export: nil, exported_at: nil
     Workers::ExportUser.perform_async(id)
@@ -325,29 +328,26 @@ def perform_export!
     else
       update exporting: false
     end
-  rescue => error
-    logger.error "Unexpected error while exporting user '#{username}': #{error.class}: #{error.message}\n" \
-                 "#{error.backtrace.first(15).join("\n")}"
+  rescue StandardError => e
+    logger.error "Unexpected error while exporting user '#{username}': #{e.class}: #{e.message}\n" \
+                 "#{e.backtrace.first(15).join("\n")}"
     update exporting: false
   end
 
   def compressed_export
     ActiveSupport::Gzip.compress Diaspora::Exporter.new(self).execute
   end
 
-  ######### Photos export ##################
-  mount_uploader :exported_photos_file, ExportedPhotos
-
   def queue_export_photos
     update exporting_photos: true, exported_photos_file: nil, exported_photos_at: nil
     Workers::ExportPhotos.perform_async(id)
   end
 
   def perform_export_photos!
     PhotoExporter.new(self).perform
-  rescue => error
-    logger.error "Unexpected error while exporting photos for '#{username}': #{error.class}: #{error.message}\n" \
-                 "#{error.backtrace.first(15).join("\n")}"
+  rescue StandardError => e
+    logger.error "Unexpected error while exporting photos for '#{username}': #{e.class}: #{e.message}\n" \
+                 "#{e.backtrace.first(15).join("\n")}"
     update exporting_photos: false
   end
 

app/services/migration_service.rb

@@ -10,9 +10,12 @@ def initialize(archive_path, new_user_name)
   end
 
   def validate
+    return unless archive_file_exists?
+
     archive_validator.validate
     raise ArchiveValidationFailed, errors.join("\n") if errors.any?
     raise MigrationAlreadyExists if AccountMigration.where(old_person: old_person).any?
+    raise SelfMigrationNotAllowed if self_import?
   end
 
   def perform!
@@ -22,6 +25,12 @@ def perform!
     remove_intermediate_file
   end
 
+  def self_import?
+    source_diaspora_id = archive_validator.archive_author_diaspora_id
+    target_diaspora_id = "#{new_user_name}#{User.diaspora_id_host}"
+    source_diaspora_id.eql?(target_diaspora_id)
+  end
+
   # when old person can't be resolved we still import data but we don't create&perform AccountMigration instance
   def only_import?
     old_person.nil?
@@ -51,6 +60,10 @@ def run_migration
     account_migration.perform!
   end
 
+  def archive_file_exists?
+    File.exist?(archive_path)
+  end
+
   def account_migration
     @account_migration ||= AccountMigration.new(
       old_person:             old_person,
@@ -120,4 +133,7 @@ class ArchiveValidationFailed < RuntimeError
 
   class MigrationAlreadyExists < RuntimeError
   end
+
+  class SelfMigrationNotAllowed < RuntimeError
+  end
 end

app/uploaders/exported_photos.rb

@@ -9,7 +9,14 @@ def store_dir
     "uploads/users"
   end
 
+  def extension_allowlist
+    %w[zip]
+  end
+
   def filename
-    "#{model.username}_photos_#{secure_token}.zip" if original_filename.present?
+    if original_filename.present?
+      extension = File.extname(@filename) if @filename
+      "#{model.username}_photos_#{secure_token}#{extension}"
+    end
   end
 end

app/uploaders/exported_user.rb

@@ -10,10 +10,13 @@ def store_dir
   end
 
   def extension_allowlist
-    %w[gz]
+    %w[gz zip json]
   end
 
   def filename
-    "#{model.username}_diaspora_data_#{secure_token}.json.gz" if original_filename.present?
+    if original_filename.present?
+      extension = File.extname(@filename) if @filename
+      "#{model.username}_data_#{secure_token}#{extension}"
+    end
   end
 end

app/uploaders/secure_uploader.rb

@@ -2,7 +2,8 @@
 
 class SecureUploader < CarrierWave::Uploader::Base
   protected
-  def secure_token(bytes = 16)
+
+  def secure_token(bytes=16)
     var = :"@#{mounted_as}_secure_token"
     model.instance_variable_get(var) or model.instance_variable_set(var, SecureRandom.urlsafe_base64(bytes))
   end

app/views/users/_edit.haml

@@ -193,8 +193,11 @@
     %hr
 
     .row
-      .col-md-6.account-data
+      .col-md-12.account-data
         %h3= t(".export_data")
+        %p If you want to migrate this account to another pod, start by exporting your data using the buttons below.
+    .row
+      .col-md-6
         %h4= t("profile")
         .form-group
           - if current_user.exporting
@@ -210,6 +213,7 @@
             = link_to t(".request_export"), export_profile_user_path, method: :post,
               class: "btn btn-default"
 
+      .col-md-6
         %h4= t("javascripts.profile.photos")
         .form-group
           - if current_user.exporting_photos
@@ -224,7 +228,24 @@
             = link_to t(".request_export_photos"), export_photos_user_path, method: :post,
               class: "btn btn-default"
 
-      .col-md-6
+    .row
+      .col-md-12
+        %h3
+          Import another account into
+          = current_user.diaspora_handle
+        %p
+          diaspora* allows you to merge an account into another, even if the accounts are in two different pods.
+          This way, you can migrate from one pod to another one.
+          For this, you need to upload into this account the data and the photo archives you have exported from the
+          settings in the other account.
+          %strong There is no way back,
+          an account merged into another one can't be restored.
+        .form-group
+          .btn.btn-primary{id: "import_account", data: {toggle: "modal", target: "#importAccountModal"}}
+            Import another account...
+          = render "import_account_modal"
+    .row
+      .col-md-12
         %h3
           = t(".close_account_text")
         .form-group

app/views/users/_import_account_modal.haml

@@ -0,0 +1,53 @@
+.modal.fade{id: "importAccountModal",
+            tabindex: "-1",
+            role: "dialog",
+            aria: {labelledby: "importAccountModalLabel", hidden: "true"}}
+  .modal-dialog
+    .modal-content
+      .modal-header
+        %button.close{type: "button", data: {dismiss: "modal"}, aria: {hidden: "true"}}
+          ×
+        %h3.modal-title{id: "importAccountModalLabel"}
+          Import another account
+        .modal-body
+          %p
+            You are about to import and merge another account into
+            = current_user.diaspora_handle
+            /.
+            Here is what's going to happen:
+          %ol
+            %li
+              Your profile (name, bio, birthday, gender...) will be updated with the content from the imported account
+            %li
+              Your settings (theme, visibility...) will be updated with those from the imported account
+            %li
+              All your aspects and contacts will be added to this account. This will be transparent for them, they
+              will be linked to that account as they were linked with the account you are importing
+            %li
+              All posts and comments you made with the imported account will be imported in that account and seen
+              as posted by this accounts by the other users
+            %li
+              %strong
+                The imported account and all its linked data will be deleted from the previous pod.
+
+          %p
+            If the pod of the account you are importing is offline, your contacts and posts will still be imported,
+            but your contacts will receive a notification as if you started sharing with them.
+          = form_for current_user, url: edit_user_path, html: {method: :put, multipart: true, class: "form-horizontal"} do |f|
+            .row
+              .col-md-6
+                %label Select the data archive:
+                = f.file_field :export, accept: "application/json, application/zip, application/gzip"
+              .col-md-6
+                %label Select the photos archive:
+                = f.file_field :exported_photos_file, accept: "application/zip"
+            %h4
+              The original account will be imported
+              %strong
+                and deleted. This cannot be undone!
+            %p
+              .clearfix
+                .btn.btn-default{data: {dismiss: "modal"}}
+                  Cancel
+                = f.submit 'Import and merge', class: "btn btn-primary.pull-right", id: "change_email_preferences"
+