create a contract event log by custom config

.gitignore

@@ -39,3 +39,8 @@ doc
 
 # docker database
 docker/data
+
+# custom configs
+config/customs/*.yml
+db/migrate/*_create_customs_*
+app/models/customs/*

Gemfile

@@ -68,6 +68,8 @@ gem 'google-protobuf', '~> 3.6'
 
 gem 'ciri-crypto', '0.1.1'
 
+gem 'ethereum.rb'
+
 # Deployment
 gem 'mina', require: false
 gem 'mina-puma', require: false

Gemfile.lock

@@ -105,6 +105,9 @@ GEM
       dotenv (= 2.5.0)
       railties (>= 3.2, < 6.0)
     erubi (1.7.1)
+    ethereum.rb (2.2)
+      activesupport (>= 4.0)
+      digest-sha3 (~> 1.1)
     execjs (2.7.0)
     factory_bot (4.10.0)
       activesupport (>= 3.0.0)
@@ -314,6 +317,7 @@ DEPENDENCIES
   daemons (~> 1.2, >= 1.2.6)
   database_cleaner
   dotenv-rails
+  ethereum.rb
   factory_bot_rails
   faraday (~> 0.15.2)
   google-protobuf (~> 3.6)
@@ -349,4 +353,4 @@ RUBY VERSION
    ruby 2.5.1p57
 
 BUNDLED WITH
-   1.16.3
+   1.16.4

app/models/event_log.rb

@@ -0,0 +1,3 @@
+class EventLog < ApplicationRecord
+  validates :name, presence: true, uniqueness: true
+end

app/models/event_log_process.rb

@@ -0,0 +1,171 @@
+class EventLogProcess
+  attr_reader :config, :table_name, :model_name, :columns, :file_name
+
+  # sync all event logs
+  def self.sync_all
+    EventLog.distinct.pluck(:name).each do |n|
+      EventLogProcess.new(n).save_event_log
+    end
+  end
+
+  # read file and initialize
+  #
+  # @param file [String] file name
+  # @return [void]
+  def initialize(file)
+    @file_name = file
+    @config = YAML.load(File.read(Rails.root.join("config", "customs", file))).with_indifferent_access
+    @table_name = config[:table_name]
+    @model_name = table_name.camelcase.singularize
+    @columns = config[:columns]
+    th_column = %w(string transactionHash)
+    @columns << th_column unless columns.include?(th_column)
+    bn_column = %w(string blockNumber)
+    @columns << bn_column unless columns.include?(bn_column)
+    EventLog.create(name: file)
+  end
+
+  # get logs from chain
+  #
+  # @return [Hash] response body
+  def get_logs
+    from_block = EventLog.find_by(name: file_name)&.block_number || "0x0"
+    filter = { fromBlock: from_block }
+    address = config[:address]
+    topics = config[:topics]
+    filter.merge!(address: address) unless address.blank?
+    filter.merge!(topics: topics) unless topics.nil? || topics.empty?
+    CitaSync::Api.get_logs(filter)
+  end
+
+  # get result of `get_logs`
+  def result
+    get_logs["result"]
+  end
+
+  # decode one event log
+  #
+  # @param log [Hash] event log
+  # @return [[Hash]] abi inputs with decoded_data
+  def decode_log(log)
+    # log = {
+    #   "address" => "0x35bd452c37d28beca42097cfd8ba671c8dd430a1",
+    #   "topics" => [
+    #     "0xe4af93ca7e370881e6f1b57272e42a3d851d3cc6d951b4f4d2e7a963914468a2",
+    #     "0x000000000000000000000000000000000000000000000000000001657f9d5fbf"
+    #   ],
+    #   "data" => "0x00000000000000000000000046a23e25df9a0f6c18729dda9ad1af3b6a1311600000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000001c68656c6c6f20776f726c64206174203135333534343433343437363700000000",
+    #   "blockHash" => "0x2bb2dab1bc4e332ca61fe15febf06a1fd09738d6304d76c5dd9b57cb46880e28",
+    #   "blockNumber" => "0xf11e2",
+    #   "transactionHash" => "0x2c12c54a55428b56fd35b5882d5087d6cf2e20a410dc3a1b6515c2ecc3f53f22",
+    #   "transactionIndex" => "0x0",
+    #   "logIndex" => "0x0",
+    #   "transactionLogIndex" => "0x0"
+    # }.with_indifferent_access
+
+    log = log.with_indifferent_access
+    decoder = Ethereum::Decoder.new
+
+    decode_info = config[:decode]
+    return if decode_info.nil?
+    abi_inputs = decode_info[:abi_inputs]
+    data_inputs = abi_inputs.select { |i| !i[:indexed] }
+    inputs = data_inputs.map { |input| Ethereum::FunctionInput.new(input) }
+    decoded_data = decoder.decode_arguments(inputs, log[:data])
+    data_inputs.each_with_index { |d, i| d[:decoded_data] = decoded_data[i] }
+
+    # topics
+    topics = log[:topics]
+    topic_inputs = abi_inputs.select { |i| i[:indexed] }.each_with_index do |a, i|
+      a[:decoded_data] = decoder.decode(a[:type], topics[i + 1])
+    end
+
+    abi_inputs
+  end
+
+  # save all event logs by `get_logs`
+  def save_event_log
+    reference = columns.map { |col| [col.second, col.third] }.each { |col| col[1] = col.first if col.last.nil? }.to_h.invert
+
+    result.each do |log|
+      attrs = log.slice(*reference.keys).transform_keys { |k| reference[k] }.merge(get_decode_attrs(log.with_indifferent_access))
+      ApplicationRecord.transaction do
+        "Customs::#{model_name}".constantize.create(attrs)
+        event_log = EventLog.find_by(name: file_name)
+        event_log&.update(block_number: attrs["blockNumber"])
+      end
+    end
+  end
+
+  # generate model and migration
+  def generate_model
+    content = <<-MODEL
+class Customs::#{model_name} < ApplicationRecord
+  self.table_name = "#{table_name}"
+  validates :transactionHash, uniqueness: true
+end
+    MODEL
+
+    file_name = "#{model_name.underscore}.rb"
+    file_path = Rails.root.join("app", "models", "customs", file_name)
+
+    if File.exist?(file_path)
+      raise "model already exist !"
+    end
+
+    generate_migration
+
+    File.open(file_path, "w") { |file| file.write(content) }
+  end
+
+  # generate a migration file to db/migrate
+  private def generate_migration
+    migration = <<-MIGRATION
+class CreateCustoms#{table_name.camelcase} < ActiveRecord::Migration[5.2]
+  def change
+    create_table :#{table_name} do |t|
+      #{columns.map { |col| "t.#{col.first} :#{col.second}" }.join("\n      ")}
+      #{get_decode_columns&.map { |col| "t.#{col.first} :#{col.second}" }&.join("\n      ")}          
+      t.timestamps
+    end
+  end
+end
+    MIGRATION
+
+    # create a migration file
+    timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S")
+    base_file_name = "create_customs_#{table_name}.rb"
+    file_name = "#{timestamp}_#{base_file_name}"
+    file_path = Rails.root.join("db", "migrate", file_name)
+    # check file exist
+    unless Dir[Rails.root.join("db", "migrate", "*.rb")].select { |f| f.end_with?(base_file_name) }.empty?
+      raise "table already exist !"
+    end
+    # check file exist
+    File.open(file_path, "w") { |file| file.write(migration) }
+  end
+
+  # for generate migration
+  #
+  # @return [[[String, String]], nil]
+  private def get_decode_columns
+    column_types = config.dig :decode, :column_types
+    names = config.dig :decode, :names
+    return if names.nil? || column_types.nil?
+    raise "decode names length not equals to decode column_names" if names.size != column_types.size
+    column_types.zip(names)
+  end
+
+  # get decoded attrs (Hash)
+  #
+  # @param log [Hash] event log
+  # @return [Hash]
+  private def get_decode_attrs(log)
+    decoded_data = decode_log(log).map { |dl| dl[:decoded_data] }
+    names = config.dig :decode, :names
+    return if names.nil? || decoded_data.nil?
+    raise "decode names length not equals to inputs" if names.size != decoded_data.size
+    Hash[names.zip(decoded_data)]
+  end
+
+end

config/customs/event_log.yml.sample

@@ -0,0 +1,33 @@
+address: "0x35bd452c37d28beca42097cfd8ba671c8dd430a1"
+topics: []
+table_name: "records"
+# type, column name, data name, no data name means data name equals to column name
+# "decode#data" means decode data
+# type can be "string", "integer", "bigint", "text"
+# we'll save ["string", "transactionHash"] and ["string", "blockNumber"] for check unique log
+columns:
+  - ["string", "data"]
+  - ["string", "blockHash", "blockHash"]
+  - ["string", "block_number", "blockNumber"]
+
+# address => string
+# string => string
+# uint256 => integer, bigint, string
+decode:
+  abi_inputs:
+    [{
+      "indexed": false,
+      "name": "_sender",
+      "type": "address"
+      }, {
+      "indexed": false,
+      "name": "_text",
+      "type": "string"
+    }, {
+      "indexed": true,
+      "name": "_time",
+      "type": "uint256"
+    }]
+  names: ["sender", "text", "time"]
+  column_types: ["string", "string", "bigint"]
+

db/migrate/20180830090621_create_event_logs.rb

@@ -0,0 +1,8 @@
+class CreateEventLogs < ActiveRecord::Migration[5.2]
+  def change
+    create_table :event_logs do |t|
+      t.string :name
+      t.string :block_number
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_08_30_054052) do
+ActiveRecord::Schema.define(version: 2018_08_30_090621) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -46,6 +46,11 @@
     t.index ["header"], name: "index_blocks_on_header", using: :gin
   end
 
+  create_table "event_logs", force: :cascade do |t|
+    t.string "name"
+    t.string "block_number"
+  end
+
   create_table "sync_errors", force: :cascade do |t|
     t.string "method"
     t.json "params"

lib/sync_control.rb

@@ -13,3 +13,11 @@
   Rails.logger = Logger.new(Rails.root.join("log", "#{Rails.env}_sync.log"))
   ::CitaSync::Persist.realtime_sync
 end
+
+# Run a process to sync event logs
+if EventLog.exists?
+  Daemons.run_proc("#{Rails.env}_event_log", options) do
+    Rails.logger = Logger.new(Rails.root.join("log", "#{Rails.env}_event_log.log"))
+    EventLogProcess.sync_all
+  end
+end

lib/tasks/event_log.rake

@@ -0,0 +1,8 @@
+namespace :event_log do
+  desc "create a event log model and migration"
+  task :create, [:file_name] => :environment do |task, args|
+    puts EventLogProcess.new(args[:file_name]).generate_model
+    puts EventLog.find_by(name: args[:file_name])&.update(block_number: nil)
+    puts `bundle exec rake db:migrate`
+  end
+end

spec/factories/event_logs.rb

@@ -0,0 +1,5 @@
+FactoryBot.define do
+  factory :event_log do
+
+  end
+end

spec/models/event_log_spec.rb

@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe EventLog, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end