Bulk import

spec/granite_orm/fields/timestamps_spec.cr

@@ -5,7 +5,7 @@ require "../../spec_helper"
 module {{adapter.capitalize.id}}
   {%
     avoid_macro_bug = 1 # https://github.com/crystal-lang/crystal/issues/5724
-    
+
     # TODO mysql timestamp support should work better
     if adapter == "pg"
       time_kind_on_read = "Time::Kind::Utc".id

spec/granite_orm/querying/find_spec.cr

@@ -60,7 +60,7 @@ module {{adapter.capitalize.id}}
     it "returns nil or raises if no result" do
       found = Parent.find? 0
       found.should be_nil
-      
+
       expect_raises(Granite::ORM::Querying::NotFound, /Couldn't find .*Parent.* with id=0/) do
         Parent.find 0
       end

spec/granite_orm/transactions/import_spec.cr

@@ -0,0 +1,17 @@
+require "../../spec_helper"
+
+{% for adapter in GraniteExample::ADAPTERS %}
+module {{adapter.capitalize.id}}
+  describe "{{ adapter.id }} .import" do
+    it "should import 3 new objects" do
+      to_import = [
+        Parent.new(name: "ImportParent1"),
+        Parent.new(name: "ImportParent2"),
+        Parent.new(name: "ImportParent3"),
+      ]
+      Parent.import(to_import)
+      Parent.all("WHERE name LIKE ?", ["Import%"]).size.should eq 3
+    end
+  end
+end
+{% end %}

spec/spec_helper.cr

@@ -1,7 +1,7 @@
 require "spec"
 
 module GraniteExample
-  ADAPTERS = ["pg","mysql","sqlite"]
+  ADAPTERS = ["pg", "mysql", "sqlite"]
 end
 
 require "../src/granite_orm"

src/adapter/base.cr

@@ -49,6 +49,9 @@ abstract class Granite::Adapter::Base
   # This will insert a row in the database and return the id generated.
   abstract def insert(table_name, fields, params, lastval) : Int64
 
+  # This will insert an array of models as one insert statement
+  abstract def import(table_name : String, primary_name : String, fields, model_array, **options)
+
   # This will update a row in the database.
   abstract def update(table_name, primary_name, fields, params)
 

src/adapter/mysql.cr

@@ -76,6 +76,44 @@ class Granite::Adapter::Mysql < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+
+    statement = String.build do |stmt|
+      stmt << "INSERT"
+      stmt << " IGNORE" if options["on_duplicate_key_ignore"]?
+      stmt << " INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        next unless model.valid?
+        stmt << '('
+        stmt << Array.new(fields.size, '?').join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+      end
+    end.chomp(',')
+
+    if update_keys = options["on_duplicate_key_update"]?
+      statement += " ON DUPLICATE KEY UPDATE "
+      update_keys.each do |key|
+        statement += "#{quote(key)}=VALUES(#{quote(key)}), "
+      end
+      statement = statement.chomp(", ")
+    end
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LAST_INSERT_ID()"
   end

src/adapter/pg.cr

@@ -76,6 +76,47 @@ class Granite::Adapter::Pg < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+    index = 0
+
+    statement = String.build do |stmt|
+      stmt << "INSERT"
+      stmt << " INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        next unless model.valid?
+        stmt << '('
+        stmt << fields.map_with_index { |_f, idx| "$#{index + idx + 1}" }.join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+        index += fields.size
+      end
+    end.chomp(',')
+
+    if update_keys = options["on_duplicate_key_update"]?
+      statement += " ON CONFLICT (#{quote(primary_name)}) DO UPDATE SET "
+      update_keys.each do |key|
+        statement += "#{quote(key)}=EXCLUDED.#{quote(key)}, "
+      end
+      statement = statement.chomp(", ")
+    elsif options["on_duplicate_key_ignore"]?
+      statement += " ON CONFLICT DO NOTHING"
+    end
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LASTVAL()"
   end

src/adapter/sqlite.cr

@@ -74,6 +74,44 @@ class Granite::Adapter::Sqlite < Granite::Adapter::Base
     end
   end
 
+  def import(table_name : String, primary_name : String, fields, model_array, **options)
+    params = [] of DB::Any
+    now = Time.now.to_utc
+    fields.reject! { |field| field === "id" } if primary_name === "id"
+
+    statement = String.build do |stmt|
+      stmt << "INSERT"
+      stmt << " IGNORE" if options["on_duplicate_key_ignore"]?
+      stmt << " INTO #{quote(table_name)} ("
+      stmt << fields.map { |field| quote(field) }.join(", ")
+      stmt << ") VALUES "
+
+      model_array.each do |model|
+        next unless model.valid?
+        model.updated_at = now if model.responds_to? :updated_at
+        model.created_at = now if model.responds_to? :created_at
+        stmt << '('
+        stmt << Array.new(fields.size, '?').join(',')
+        params.concat fields.map { |field| model.to_h[field] }
+        stmt << "),"
+      end
+    end.chomp(',')
+
+    if update_keys = options["on_duplicate_key_update"]?
+      statement += " ON DUPLICATE KEY UPDATE "
+      update_keys.each do |key|
+        statement += "#{quote(key)}=VALUES(#{quote(key)}), "
+      end
+      statement = statement.chomp(", ")
+    end
+
+    log statement, params
+
+    open do |db|
+      db.exec statement, params
+    end
+  end
+
   private def last_val
     return "SELECT LAST_INSERT_ROWID()"
   end

src/granite_orm/associations.cr

@@ -33,7 +33,7 @@ module Granite::ORM::Associations
       {{children_class}}.all(query, id)
     end
   end
-  
+
   # define getter for related children
   macro has_many(children_table, through)
     def {{children_table.id}}

src/granite_orm/querying.cr

@@ -1,6 +1,6 @@
 module Granite::ORM::Querying
   class NotFound < Exception
-  end  
+  end
 
   macro extended
     macro __process_querying

src/granite_orm/transactions.cr

@@ -7,9 +7,20 @@ module Granite::ORM::Transactions
     @updated_at : Time?
     @created_at : Time?
 
+    # The import class method will run a batch INSERT statement for each model in the array
+    # the array must contain only one model class
+    # invalid model records will be skipped
+    def self.import(model_array : Array(self), **options)
+      begin
+      @@adapter.import(table_name, primary_name, fields, model_array, **options)
+      rescue err
+        raise DB::Error.new(err.message)
+      end
+    end
+
     # The save method will check to see if the primary exists yet. If it does it
     # will call the update method, otherwise it will call the create method.
-    # This will update the timestamps apropriately.
+    # This will update the timestamps appropriately.
     def save
       return false unless valid?
 
@@ -91,7 +102,7 @@ module Granite::ORM::Transactions
     end
   end
 
-  module ClassMethods 
+  module ClassMethods
     def create(**args)
       create(args.to_h)
     end

src/granite_orm/validators.cr

@@ -8,7 +8,7 @@ require "./error"
 #   !user.name.to_s.blank?
 # end
 #
-# validate :name, "can't be blank", -> (user : User) do
+# validate :name, "can't be blank", ->(user : User) do
 #   !user.name.to_s.blank?
 # end
 #