Merge pull request #109 from hogelog/disable-online-sync

Remove sync data source feature from web setting view

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 0.6.0 (Not released)
+- Remove sync data source feature from web setting view
+- Refactoring
+
 ## 0.5.0
 - Upgrade to Rails v5.2 [#99](https://github.com/hogelog/dmemo/pull/99)
 - Upgrade many dependencies

app/batches/synchronize_data_sources.rb

@@ -1,7 +1,93 @@
 class SynchronizeDataSources
+  DEFAULT_FETCH_ROWS_LIMIT = 20
+
   def self.run
     DataSource.all.find_each do |data_source|
-      DatabaseMemo.import_data_source!(data_source.id)
+      import_data_source!(data_source)
     end
   end
+
+  def self.import_data_source!(data_source)
+    data_source.reset_data_source_tables!
+
+    db_memo = DatabaseMemo.find_or_create_by!(name: data_source.name)
+    schema_memos = db_memo.schema_memos.includes(table_memos: [:column_memos, :raw_dataset]).to_a
+    schema_memos.each {|memo| memo.linked = false }
+
+    all_table_memos = schema_memos.map(&:table_memos).map(&:to_a).flatten
+    all_table_memos.each {|memo| memo.linked = false }
+
+    data_source_tables = data_source.data_source_tables
+
+    data_source_tables.group_by(&:schema_name).each do |schema_name, source_tables|
+      schema_memo = schema_memos.find {|memo| memo.name == schema_name } || db_memo.schema_memos.create!(name: schema_name )
+      schema_memo.linked = true
+      table_memos = all_table_memos.select {|memo| memo.schema_memo_id == schema_memo.id }
+
+      source_tables.each do |source_table|
+        begin
+          import_table_memo!(schema_memo, table_memos, source_table)
+        rescue DataSource::ConnectionBad => e
+          Rails.logger.error e
+        end
+      end
+    end
+    schema_memos.each {|memo| memo.save! if memo.changed? }
+    all_table_memos.each {|memo| memo.save! if memo.changed? }
+  rescue Mysql2::Error, PG::Error => e
+    raise DataSource::ConnectionBad.new(e)
+  end
+
+  def self.import_table_memo!(schema_memo, table_memos, source_table)
+    table_name = source_table.table_name
+    table_memo = table_memos.find {|memo| memo.name == table_name } || schema_memo.table_memos.create!(name: table_name )
+    table_memo.linked = true
+    column_memos = table_memo.column_memos.to_a
+
+    columns = source_table.columns
+    adapter = source_table.source_base_class.connection.pool.connections.first
+
+    import_table_memo_raw_dataset!(table_memo, source_table, columns)
+
+    column_names = columns.map(&:name)
+    column_memos.reject {|memo| column_names.include?(memo.name) }.each {|memo| memo.update!(linked: false) }
+
+    columns.each_with_index do |column, position|
+      column_memo = column_memos.find {|memo| memo.name == column.name } || table_memo.column_memos.build(name: column.name)
+      column_memo.linked = true
+      column_memo.assign_attributes(sql_type: column.sql_type, default: adapter.quote(column.default), nullable: column.null, position: position)
+      column_memo.save! if column_memo.changed?
+    end
+
+    table_memo.save! if table_memo.changed?
+  rescue Mysql2::Error, PG::Error => e
+    raise DataSource::ConnectionBad.new(e)
+  end
+  private_class_method :import_table_memo!
+
+  def self.import_table_memo_raw_dataset!(table_memo, source_table, columns)
+    table_count = source_table.fetch_count
+    if table_memo.raw_dataset
+      unless table_memo.raw_dataset.same_columns?(columns) && (table_memo.raw_dataset.count >= DEFAULT_FETCH_ROWS_LIMIT || table_memo.raw_dataset.count == table_count)
+        import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
+      end
+      table_memo.raw_dataset.update!(count: table_count)
+    else
+      table_memo.create_raw_dataset!(count: table_count)
+      import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
+    end
+  end
+  private_class_method :import_table_memo_raw_dataset!
+
+  def self.import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
+    table_memo.raw_dataset.columns.delete_all
+    dataset_columns = columns.map.with_index do |column, position|
+      table_memo.raw_dataset.columns.create!(name: column.name, sql_type: column.sql_type, position: position)
+    end
+    table_memo.raw_dataset.rows.delete_all
+    source_table.fetch_rows(DEFAULT_FETCH_ROWS_LIMIT).each do |row|
+      table_memo.raw_dataset.rows.create!(row: row.map.with_index{|value, i| dataset_columns[i].format_value(value) })
+    end
+  end
+  private_class_method :import_table_memo_raw_dataset_rows!
 end

app/models/database_memo.rb

@@ -2,8 +2,6 @@ class DatabaseMemo < ApplicationRecord
   include TextDiff
   include DescriptionLogger
 
-  DEFAULT_FETCH_ROWS_LIMIT = 20
-
   scope :id_or_name, ->(id, name) { where("database_memos.id = ? OR database_memos.name = ?", id.to_i, name) }
 
   has_many :schema_memos, dependent: :destroy
@@ -13,38 +11,6 @@ class DatabaseMemo < ApplicationRecord
 
   validates :name, presence: true
 
-  def self.import_data_source!(data_source_id)
-    data_source = DataSource.find(data_source_id)
-    data_source.reset_data_source_tables!
-
-    db_memo = find_or_create_by!(name: data_source.name)
-    schema_memos = db_memo.schema_memos.includes(table_memos: [:column_memos, :raw_dataset]).to_a
-    schema_memos.each {|memo| memo.linked = false }
-
-    all_table_memos = schema_memos.map(&:table_memos).map(&:to_a).flatten
-    all_table_memos.each {|memo| memo.linked = false }
-
-    data_source_tables = data_source.data_source_tables
-
-    data_source_tables.group_by(&:schema_name).each do |schema_name, source_tables|
-      schema_memo = schema_memos.find {|memo| memo.name == schema_name } || db_memo.schema_memos.create!(name: schema_name )
-      schema_memo.linked = true
-      table_memos = all_table_memos.select {|memo| memo.schema_memo_id == schema_memo.id }
-
-      source_tables.each do |source_table|
-        begin
-          import_table_memo!(schema_memo, table_memos, source_table)
-        rescue DataSource::ConnectionBad => e
-          Rails.logger.error e
-        end
-      end
-    end
-    schema_memos.each {|memo| memo.save! if memo.changed? }
-    all_table_memos.each {|memo| memo.save! if memo.changed? }
-  rescue Mysql2::Error, PG::Error => e
-    raise DataSource::ConnectionBad.new(e)
-  end
-
   def linked?
     data_source.present?
   end
@@ -56,57 +22,4 @@ def single_schema?
   def display_order
     [linked? ? 0 : 1, name]
   end
-
-  def self.import_table_memo!(schema_memo, table_memos, source_table)
-    table_name = source_table.table_name
-    table_memo = table_memos.find {|memo| memo.name == table_name } || schema_memo.table_memos.create!(name: table_name )
-    table_memo.linked = true
-    column_memos = table_memo.column_memos.to_a
-
-    columns = source_table.columns
-    adapter = source_table.source_base_class.connection.pool.connections.first
-
-    import_table_memo_raw_dataset!(table_memo, source_table, columns)
-
-    column_names = columns.map(&:name)
-    column_memos.reject {|memo| column_names.include?(memo.name) }.each {|memo| memo.update!(linked: false) }
-
-    columns.each_with_index do |column, position|
-      column_memo = column_memos.find {|memo| memo.name == column.name } || table_memo.column_memos.build(name: column.name)
-      column_memo.linked = true
-      column_memo.assign_attributes(sql_type: column.sql_type, default: adapter.quote(column.default), nullable: column.null, position: position)
-      column_memo.save! if column_memo.changed?
-    end
-
-    table_memo.save! if table_memo.changed?
-  rescue Mysql2::Error, PG::Error => e
-    raise DataSource::ConnectionBad.new(e)
-  end
-  private_class_method :import_table_memo!
-
-  def self.import_table_memo_raw_dataset!(table_memo, source_table, columns)
-    table_count = source_table.fetch_count
-    if table_memo.raw_dataset
-      unless table_memo.raw_dataset.same_columns?(columns) && (table_memo.raw_dataset.count >= DEFAULT_FETCH_ROWS_LIMIT || table_memo.raw_dataset.count == table_count)
-        import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
-      end
-      table_memo.raw_dataset.update!(count: table_count)
-    else
-      table_memo.create_raw_dataset!(count: table_count)
-      import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
-    end
-  end
-  private_class_method :import_table_memo_raw_dataset!
-
-  def self.import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
-    table_memo.raw_dataset.columns.delete_all
-    dataset_columns = columns.map.with_index do |column, position|
-      table_memo.raw_dataset.columns.create!(name: column.name, sql_type: column.sql_type, position: position)
-    end
-    table_memo.raw_dataset.rows.delete_all
-    source_table.fetch_rows(DEFAULT_FETCH_ROWS_LIMIT).each do |row|
-      table_memo.raw_dataset.rows.create!(row: row.map.with_index{|value, i| dataset_columns[i].format_value(value) })
-    end
-  end
-  private_class_method :import_table_memo_raw_dataset_rows!
 end

app/views/settings/_data_sources.html.haml

@@ -35,6 +35,3 @@
               = link_to data_source_path(data_source), method: :delete, data: { confirm: "Delete data source #{data_source.name}?" } do
                 %button.btn.btn-default
                   %i.fa.fa-trash
-              = form_tag data_source_synchronized_database_path(data_source.id), method: :patch, class: "inline" do
-                %button.btn.btn-default
-                  %i.fa.fa-refresh

config/routes.rb

@@ -2,9 +2,7 @@
   root "top#show"
 
   resource :setting, only: %w(show)
-  resources :data_sources do
-    resource :synchronized_database, only: %w(update)
-  end
+  resources :data_sources
 
   resources :masked_data, except: %w(edit update)
   resources :ignored_tables, except: %w(edit update)

spec/batches/synchronize_data_sources_spec.rb

@@ -1,10 +1,11 @@
 require "rails_helper"
 
 describe SynchronizeDataSources do
-  let!(:data_source) { FactoryBot.create(:data_source, name: "dmemo", description: "") }
   let(:batch) { SynchronizeDataSources }
 
   describe ".run" do
+    let!(:data_source) { FactoryBot.create(:data_source, name: "dmemo", description: "") }
+
     it "synchronize data_source" do
       expect(DatabaseMemo.count).to eq(0)
       expect(TableMemoRawDataset.count).to eq(0)
@@ -40,4 +41,52 @@
       end
     end
   end
+
+  describe ".import_data_source" do
+    let(:data_source) { FactoryBot.create(:data_source) }
+
+    describe ".import_data_source!" do
+      it "synchronize data source" do
+        FactoryBot.create(:keyword, name: "tempura")
+
+        expect(DatabaseMemo.count).to eq(0)
+        batch.import_data_source!(data_source)
+
+        database_memo = DatabaseMemo.take
+        expect(database_memo.name).to eq(data_source.name)
+
+        keywords_table = TableMemo.find_by(name: "keywords")
+        expect(keywords_table.raw_dataset.count).to eq(1)
+        expect(keywords_table.raw_dataset.rows.size).to eq(1)
+
+        FactoryBot.create(:keyword, name: "sushi")
+
+        batch.import_data_source!(data_source)
+        expect(keywords_table.reload.raw_dataset.count).to eq(2)
+        expect(keywords_table.raw_dataset.rows.size).to eq(2)
+      end
+
+      context "when columns doesn't changed" do
+        before do
+          stub_const("SynchronizeDataSources::DEFAULT_FETCH_ROWS_LIMIT", 4)
+          4.times{|i| FactoryBot.create(:keyword, name: "sushi #{i}") }
+          batch.import_data_source!(data_source)
+        end
+
+        it "skip update raw_dataset.rows" do
+          keywords_table = TableMemo.find_by(name: "keywords")
+          expect(keywords_table.raw_dataset.count).to eq(4)
+          before_row_ids = keywords_table.raw_dataset.rows.order(:id).pluck(:id)
+
+          FactoryBot.create(:keyword, name: "tempura")
+          batch.import_data_source!(data_source)
+          keywords_table.reload
+          after_row_ids = keywords_table.raw_dataset.rows.order(:id).pluck(:id)
+
+          expect(keywords_table.raw_dataset.count).to eq(5)
+          expect(after_row_ids).to eq(before_row_ids)
+        end
+      end
+    end
+  end
 end