archive: further updates to unarchiver

app/models/autotest_setting.rb

@@ -4,6 +4,10 @@ class AutotestSetting < ApplicationRecord
   validates :url, presence: true
   before_create :register_autotester
 
+  # Column name used to identify this record if the primary identifier (id) cannot be relied on.
+  # For example, when unarchiving courses.
+  IDENTIFIER = 'url'.freeze
+
   private
 
   def register_autotester

app/models/lti_line_item.rb

@@ -1,5 +1,8 @@
 class LtiLineItem < ApplicationRecord
   belongs_to :assessment
   belongs_to :lti_deployment
+  has_one :course, through: :assessment
+
   validates :assessment, uniqueness: { scope: :lti_deployment_id }
+  validate :courses_should_match
 end

app/models/lti_service.rb

@@ -6,4 +6,5 @@ class LtiService < ApplicationRecord
   belongs_to :lti_deployment
   validates :service_type, inclusion: { in: LTI_SERVICES.values }
   validates :service_type, uniqueness: { scope: :lti_deployment_id }
+  has_one :course, through: :lti_deployment
 end

app/models/user.rb

@@ -38,6 +38,10 @@ class User < ApplicationRecord
   AUTHENTICATE_BAD_PLATFORM = 'bad_platform'.freeze
   AUTHENTICATE_BAD_CHAR = 'bad_char'.freeze
 
+  # Column name used to identify this record if the primary identifier (id) cannot be relied on.
+  # For example, when unarchiving courses.
+  IDENTIFIER = 'user_name'.freeze
+
   # Authenticates login against its password
   # through a script specified by Settings.validate_file
   def self.authenticate(login, password, ip: nil)

lib/archive_tools/course_archiver.rb

@@ -53,11 +53,11 @@ def archive_data_files(klass, ids, archive_dir)
     # This method may raise an error if no defined order can be determined, ie. if there is a dependency cycle
     def load_order
       Rails.application.eager_load!
-      dependencies = ApplicationRecord.descendants.reject(&:abstract_class).map do |r|
-        [r.table_name,
-         r.reflect_on_all_associations(:belongs_to).map do |a|
+      dependencies = ApplicationRecord.descendants.reject(&:abstract_class).map do |klass|
+        [klass.table_name,
+         klass.reflect_on_all_associations(:belongs_to).map do |a|
            if a.polymorphic?
-             r.pluck(a.foreign_type).map(&:constantize).map(&:table_name)
+             klass.pluck(a.foreign_type).map(&:constantize).map(&:table_name)
            else
              a.klass.table_name
            end
@@ -190,7 +190,10 @@ def archive(course_name)
           end
           next if ids.empty?
           File.open(archive_dir + "db/#{table_name}.csv", 'w') do |f|
-            query = "COPY (SELECT * FROM #{table_name} WHERE id IN (#{ids.to_a.join(', ')})) TO STDOUT CSV HEADER"
+            # Order by id to ensure that rows with foreign key references to other rows in the same table get
+            # created in the right order.
+            query = "COPY (SELECT * FROM #{table_name} WHERE id IN (#{ids.to_a.join(', ')}) ORDER BY id) " \
+                    'TO STDOUT CSV HEADER'
             ActiveRecord::Base.connection.raw_connection.copy_data(query) do
               while (row = ActiveRecord::Base.connection.raw_connection.get_copy_data)
                 f.write row
@@ -268,7 +271,7 @@ def unarchive(archive_file, tmp_db_url: nil)
               klass = parent_class
             else
               # Get the subclass if the current table uses single table inheritance
-              klass = parent_class.new(parent_class.inheritance_column => subclass_name).class
+              klass = subclass_name.constantize
             end
 
             # transform columns with an enum value to the enum key so that ActiveRecord can use it
@@ -309,35 +312,48 @@ def unarchive(archive_file, tmp_db_url: nil)
               end
             end
 
-            # nullify the id so that it can be assigned a new one on save
-            record.id = nil
-            if record.save
-              # save the new id so that future records with associations to this record can refer to its new id
-              new_ids[table_name][row['id']] = record.id
+            # nullify the id so that it can be assigned a new one.
+            # insert is used so that callbacks and validations are not run
+            result = klass.insert(record.attributes.except('id'),
+                                  returning: :id,
+                                  record_timestamps: false)
+            new_id = result.rows.flatten.first
+            if !new_id.nil?
+              new_ids[table_name][row['id']] = new_id
               unless old_file_location.nil? || new_file_location.nil?
                 # copy any associated files from the archived location to the new location on disk
                 FileUtils.mkdir_p(File.dirname(new_file_location))
                 FileUtils.cp_r(old_file_location, new_file_location)
                 new_file_locations << new_file_location
               end
             elsif record.respond_to?(:course)
-              warn "Unable to create record #{record.inspect}\nError(s): #{record.errors.full_messages.join(', ')}"
+              warn "Unable to create record due to database conflict: #{record.inspect}"
               errors_reported = true
             else
-              taken_attrs = record.errors
-                                  .select { |e| e.type == :taken }
-                                  .map { |e| [e.attribute, e.options[:value]] }
-                                  .to_h
-              old_record = record.class.find_by(taken_attrs)
+              if klass.const_defined?(:IDENTIFIER)
+                old_record = klass.find_by(record.attributes.slice(klass::IDENTIFIER))
+              else
+                old_record = nil
+              end
               if old_record.nil?
-                warn "Unable to create record #{record.inspect}\nError(s): #{record.errors.full_messages.join(', ')}"
+                warn "Unable to create record due to database conflict: #{record.inspect}"
                 errors_reported = true
               else
                 new_ids[table_name][row['id']] = old_record.id
               end
             end
           end
         end
+        # validate all new records now that everything has been created
+        new_ids.each do |table_name, ids|
+          ids.each_value do |id|
+            record = table_class_mapping[table_name].find(id)
+            unless record.validate
+              warn "Record #{record.inspect} is invalid\nError(s): #{record.errors.full_messages.join(', ')}"
+              errors_reported = true
+            end
+          end
+        end
       ensure
         if errors_reported
           warn "Do you want to commit all changes even though there were some errors reported? Type 'yes' to confirm."