remove migration and model files for event logs...

app/models/event_log_process.rb

@@ -1,9 +1,13 @@
 class EventLogProcess
-  attr_reader :config, :table_name, :model_name, :columns, :file_name
+  attr_reader :config, :table_name, :model_name, :columns, :file_name, :real_table_name
+
+  # all tables add prefix with "event_log_"
+  TABLE_PREFIX = "event_log_"
 
   # sync all event logs
   def self.sync_all
-    EventLog.distinct.pluck(:name).each do |n|
+    event_tables = ApplicationRecord.connection.tables.select { |t| t.start_with?(TABLE_PREFIX) }
+    event_tables.map {|n| n[10..-1]}.each do |n|
       EventLogProcess.new(n).save_event_log
     end
   end
@@ -13,23 +17,49 @@ def self.sync_all
   # @param file [String] file name
   # @return [void]
   def initialize(file)
-    @file_name = file
-    @config = YAML.load(File.read(Rails.root.join("config", "customs", file))).with_indifferent_access
-    @table_name = config[:table_name]
+    @file_name = "#{file}.yml"
+    @config = YAML.load(File.read(Rails.root.join("config", "customs", file_name))).with_indifferent_access
+    @table_name = file
     @model_name = table_name.camelcase.singularize
     @columns = config[:columns]
-    th_column = %w(string transactionHash)
-    @columns << th_column unless columns.include?(th_column)
-    bn_column = %w(string blockNumber)
+    # th_column = %w(varchar transactionHash)
+    # @columns << th_column unless columns.include?(th_column)
+    bn_column = %w(varchar blockNumber)
     @columns << bn_column unless columns.include?(bn_column)
-    EventLog.create(name: file)
+    bn_decimal_column = %w(integer blockNumberInDecimal)
+    @columns << bn_decimal_column unless columns.include?(bn_decimal_column)
+    @real_table_name = TABLE_PREFIX + table_name
+  end
+
+  def create_table
+    # raise error if table already exists
+    raise "table already exists" if ApplicationRecord.connection.table_exists?(real_table_name)
+
+    sql = <<~SQL
+          create table if not exists #{real_table_name}
+          (
+            id bigserial not null constraint #{real_table_name}_pkey primary key,
+            "created_at" timestamp not null,
+            "updated_at" timestamp not null,
+            "transactionHash" varchar not null UNIQUE,  
+            #{columns.map { |col| "\"#{col.second}\" #{col.first}," }.join("\n  ")}
+            #{get_decode_columns&.map { |col| "\"#{col.second}\" #{col.first}," }&.join("\n  ")}
+          )
+    SQL
+    lines = sql.lines
+    lines[-2].gsub!(/,/, '')
+    sql = lines.join
+    # execute to create table
+    ApplicationRecord.connection.execute(sql)
   end
 
   # get logs from chain
   #
   # @return [Hash] response body
   def get_logs
-    from_block = EventLog.find_by(name: file_name)&.block_number || "0x0"
+    current_block_number = get_current_block_number
+    from_block = HexUtils.to_hex(current_block_number.nil? ? 0 : current_block_number + 1)
+
     filter = { fromBlock: from_block }
     address = config[:address]
     topics = config[:topics]
@@ -38,6 +68,14 @@ def get_logs
     CitaSync::Api.get_logs(filter)
   end
 
+  private def get_current_block_number
+    attr_name = "blockNumberInDecimal"
+    sql = %Q{ SELECT "#{attr_name}" from #{real_table_name} ORDER BY "#{attr_name}" DESC LIMIT 1 }
+    first = ApplicationRecord.connection.execute(sql).to_set.first
+    return if first.nil?
+    first[attr_name]
+  end
+
   # get result of `get_logs`
   def result
     get_logs["result"]
@@ -48,21 +86,6 @@ def result
   # @param log [Hash] event log
   # @return [[Hash]] abi inputs with decoded_data
   def decode_log(log)
-    # log = {
-    #   "address" => "0x35bd452c37d28beca42097cfd8ba671c8dd430a1",
-    #   "topics" => [
-    #     "0xe4af93ca7e370881e6f1b57272e42a3d851d3cc6d951b4f4d2e7a963914468a2",
-    #     "0x000000000000000000000000000000000000000000000000000001657f9d5fbf"
-    #   ],
-    #   "data" => "0x00000000000000000000000046a23e25df9a0f6c18729dda9ad1af3b6a1311600000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000001c68656c6c6f20776f726c64206174203135333534343433343437363700000000",
-    #   "blockHash" => "0x2bb2dab1bc4e332ca61fe15febf06a1fd09738d6304d76c5dd9b57cb46880e28",
-    #   "blockNumber" => "0xf11e2",
-    #   "transactionHash" => "0x2c12c54a55428b56fd35b5882d5087d6cf2e20a410dc3a1b6515c2ecc3f53f22",
-    #   "transactionIndex" => "0x0",
-    #   "logIndex" => "0x0",
-    #   "transactionLogIndex" => "0x0"
-    # }.with_indifferent_access
-
     log = log.with_indifferent_access
     decoder = Ethereum::Decoder.new
 
@@ -84,65 +107,46 @@ def decode_log(log)
   end
 
   # save all event logs by `get_logs`
+  #
+  # @return [void]
   def save_event_log
-    reference = columns.map { |col| [col.second, col.third] }.each { |col| col[1] = col.first if col.last.nil? }.to_h.invert
+    dynamic_model
+
+    reference = (columns << %w(varchar transactionHash)).map { |col| [col.second, col.third] }.each { |col| col[1] = col.first if col.last.nil? }.to_h.invert
 
     result.each do |log|
-      attrs = log.slice(*reference.keys).transform_keys { |k| reference[k] }.merge(get_decode_attrs(log.with_indifferent_access))
+      block_number = log["blockNumber"]
+      block_number_in_decimal = HexUtils.to_decimal(block_number)
+      attrs = log.slice(*reference.keys).transform_keys { |k| reference[k] }.merge(get_decode_attrs(log.with_indifferent_access)).merge({ blockNumberInDecimal: block_number_in_decimal })
       ApplicationRecord.transaction do
-        "Customs::#{model_name}".constantize.create(attrs)
+        "EventLogProcess::Customs::#{model_name}".constantize.create(attrs)
         event_log = EventLog.find_by(name: file_name)
         event_log&.update(block_number: attrs["blockNumber"])
       end
     end
   end
 
-  # generate model and migration
-  def generate_model
-    content = <<-MODEL
-class Customs::#{model_name} < ApplicationRecord
-  self.table_name = "#{table_name}"
-  validates :transactionHash, uniqueness: true
-end
-    MODEL
-
-    file_name = "#{model_name.underscore}.rb"
-    file_path = Rails.root.join("app", "models", "customs", file_name)
-
-    if File.exist?(file_path)
-      raise "model already exist !"
-    end
-
-    generate_migration
-
-    File.open(file_path, "w") { |file| file.write(content) }
+  # return the dynamic model of this table
+  def get_model
+    dynamic_model
+    "EventLogProcess::Customs::#{model_name}".constantize
   end
 
-  # generate a migration file to db/migrate
-  private def generate_migration
-    migration = <<-MIGRATION
-class CreateCustoms#{table_name.camelcase} < ActiveRecord::Migration[5.2]
-  def change
-    create_table :#{table_name} do |t|
-      #{columns.map { |col| "t.#{col.first} :#{col.second}" }.join("\n      ")}
-      #{get_decode_columns&.map { |col| "t.#{col.first} :#{col.second}" }&.join("\n      ")}          
-      t.timestamps
-    end
+  class CustomsBaseClass < ApplicationRecord
   end
-end
-    MIGRATION
-
-    # create a migration file
-    timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
-    base_file_name = "create_customs_#{table_name}.rb"
-    file_name = "#{timestamp}_#{base_file_name}"
-    file_path = Rails.root.join("db", "migrate", file_name)
-    # check file exist
-    unless Dir[Rails.root.join("db", "migrate", "*.rb")].select { |f| f.end_with?(base_file_name) }.empty?
-      raise "table already exist !"
+  module Customs
+  end
+  private def dynamic_model
+    class_name = "#{model_name}"
+
+    return if EventLogProcess::Customs.const_defined?(class_name)
+
+    rtn = real_table_name
+    klass = Class.new(CustomsBaseClass) do
+      self.table_name = "#{rtn}"
     end
-    # check file exist
-    File.open(file_path, "w") { |file| file.write(migration) }
+
+    Customs.const_set class_name, klass
   end
 
   # for generate migration
@@ -168,4 +172,30 @@ def change
     Hash[names.zip(decoded_data)]
   end
 
+  # helper method, mock a log
+  #
+  # @return [Hash] a hash of event log
+  private def mock_log
+    {
+      "address" => "0x35bd452c37d28beca42097cfd8ba671c8dd430a1",
+      "topics" => [
+        "0xe4af93ca7e370881e6f1b57272e42a3d851d3cc6d951b4f4d2e7a963914468a2",
+        "0x000000000000000000000000000000000000000000000000000001657f9d5fbf"
+      ],
+      "data" => "0x00000000000000000000000046a23e25df9a0f6c18729dda9ad1af3b6a1311600000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000001c68656c6c6f20776f726c64206174203135333534343433343437363700000000",
+      "blockHash" => "0x2bb2dab1bc4e332ca61fe15febf06a1fd09738d6304d76c5dd9b57cb46880e28",
+      "blockNumber" => "0xf11e2",
+      "transactionHash" => "0x2c12c54a55428b56fd35b5882d5087d6cf2e20a410dc3a1b6515c2ecc3f53f22",
+      "transactionIndex" => "0x0",
+      "logIndex" => "0x0",
+      "transactionLogIndex" => "0x0"
+    }.with_indifferent_access
+  end
+
+  # helper method, use it carefully
+  private def drop_table
+    sql = %Q{ DROP TABLE #{real_table_name}; }
+    ApplicationRecord.connection.execute(sql)
+  end
+
 end

app/utils/hex_utils.rb

@@ -13,7 +13,7 @@ def to_hex(number)
     # @param str [String] hex string
     # @return [Integer] decimal number
     def to_decimal(str)
-      str.hex
+      str&.hex
     end
   end
 end

config/customs/event_log.yml.sample

@@ -1,33 +1,36 @@
+# your file name is you table name, for example, this table name is "event_log"
+
+# user address and topics to select event logs(jsonrpc getLogs)
 address: "0x35bd452c37d28beca42097cfd8ba671c8dd430a1"
 topics: []
-table_name: "records"
 # type, column name, data name, no data name means data name equals to column name
 # "decode#data" means decode data
-# type can be "string", "integer", "bigint", "text"
-# we'll save ["string", "transactionHash"] and ["string", "blockNumber"] for check unique log
+# Postgres database types
+# type can be "varchar", "integer", "bigint", "text", "json" and so on...
+# we'll save ["varchar", "transactionHash"] and ["varchar", "blockNumber"] for check unique log
 columns:
-  - ["string", "data"]
-  - ["string", "blockHash", "blockHash"]
-  - ["string", "block_number", "blockNumber"]
+- ["varchar", "data"] # preferred
+- ["varchar", "blockHash", "blockHash"]
+- ["varchar", "transaction_index", "transactionIndex"]
 
-# address => string
-# string => string
-# uint256 => integer, bigint, string
+# address => varchar
+# string => varchar
+# uint256 => integer, bigint, varchar
 decode:
   abi_inputs:
     [{
-      "indexed": false,
-      "name": "_sender",
-      "type": "address"
-      }, {
-      "indexed": false,
-      "name": "_text",
-      "type": "string"
-    }, {
-      "indexed": true,
-      "name": "_time",
-      "type": "uint256"
-    }]
+       "indexed": false,
+       "name": "_sender",
+       "type": "address"
+     }, {
+       "indexed": false,
+       "name": "_text",
+       "type": "string"
+     }, {
+       "indexed": true,
+       "name": "_time",
+       "type": "uint256"
+     }]
   names: ["sender", "text", "time"]
-  column_types: ["string", "string", "bigint"]
+  column_types: ["varchar", "varchar", "bigint"]
 

lib/tasks/event_log.rake

@@ -1,8 +1,6 @@
 namespace :event_log do
   desc "create a event log model and migration"
   task :create, [:file_name] => :environment do |task, args|
-    puts EventLogProcess.new(args[:file_name]).generate_model
-    puts EventLog.find_by(name: args[:file_name])&.update(block_number: nil)
-    puts `bundle exec rake db:migrate`
+    puts EventLogProcess.new(args[:file_name]).create_table
   end
 end