Merge pull request #118 from ragi256/modify_redshift_adapter_to_suppo…

…rt_late_binding_view

Support late binding view

app/batches/synchronize_data_sources.rb

@@ -42,6 +42,7 @@ def self.import_table_memo!(schema_memo, table_memos, source_table)
 
     columns = source_table.columns
     import_table_memo_raw_dataset!(table_memo, source_table, columns)
+    import_view_query!(table_memo, source_table)
 
     column_names = columns.map(&:name)
     column_memos.reject {|memo| column_names.include?(memo.name) }.each {|memo| memo.update!(linked: false) }
@@ -82,4 +83,17 @@ def self.import_table_memo_raw_dataset_rows!(table_memo, source_table, columns)
     end
   end
   private_class_method :import_table_memo_raw_dataset_rows!
+
+  def self.import_view_query!(table_memo, source_table)
+    query = source_table.fetch_view_query
+    query_plan = source_table.fetch_view_query_plan
+    if query && query_plan
+      ViewMetaDatum.find_or_initialize_by(table_memo_id: table_memo.id) do |meta_data|
+        meta_data.query = query
+        meta_data.explain = query_plan
+        meta_data.save
+      end
+    end
+  end
+  private_class_method :import_view_query!
 end

app/controllers/table_memos_controller.rb

@@ -15,6 +15,7 @@ def show(database_name, schema_name, name)
       @raw_dataset_columns = @raw_dataset.columns.order(:position)
       @raw_dataset_rows = @raw_dataset.rows.pluck(:row)
     end
+    @view_meta_data = @table_memo.view_meta_data
   end
 
   def edit(id)

app/models/data_source_adapters/base.rb

@@ -20,6 +20,14 @@ def fetch_count(table)
       nil
     end
 
+    def fetch_view_query(view)
+      nil
+    end
+
+    def fetch_view_query_plan(query)
+      raise NotImplementedError
+    end
+
     def reset!
       raise NotImplementedError
     end

app/models/data_source_adapters/redshift_adapter.rb

@@ -6,9 +6,14 @@ def fetch_table_names
       query_result = source_base_class.connection.query(<<~SQL, 'SCHEMA')
         SELECT table_schema, table_name, table_type
         FROM (
-          SELECT table_schema, table_name, table_type FROM svv_tables WHERE table_schema = ANY (current_schemas(false)) or table_type = 'EXTERNAL TABLE'
+          SELECT table_schema, table_name, table_type
+          FROM svv_tables
+          WHERE table_schema = ANY (current_schemas(false)) or table_type = 'EXTERNAL TABLE'
+
           UNION
-          SELECT schemaname as table_schema, viewname AS table_name, 'VIEW' FROM pg_views WHERE schemaname = ANY (current_schemas(false))
+
+          SELECT DISTINCT view_schema, view_name, 'LATE BINDING'
+          FROM pg_get_late_binding_view_cols() cols(view_schema name, view_name name, col_name name, col_type varchar, col_num int)
         ) tables
         ORDER BY table_schema, table_name;
       SQL
@@ -18,32 +23,43 @@ def fetch_table_names
       @base_table_names = table_groups['BASE TABLE'] || []
       @external_table_names = table_groups['EXTERNAL TABLE'] || []
       @view_names = table_groups['VIEW'] || []
+      @late_binding_view_names = table_groups['LATE BINDING'] || []
 
-      (@base_table_names + @external_table_names + @view_names).uniq
+      (@base_table_names + @external_table_names + @view_names + @late_binding_view_names).uniq
     rescue ActiveRecord::ActiveRecordError, PG::Error => e
       raise DataSource::ConnectionBad.new(e)
     end
 
     def fetch_columns(table)
       adapter = connection.pool.connection
-      if spectrum?(table)
-        connection.query(<<~SQL, 'COLUMN').map { |name, sql_type| Column.new(name, sql_type, "NULL", true) }
-          SELECT columnname, external_type FROM svv_external_columns WHERE tablename = '#{table.table_name}';
-        SQL
-      else
-        connection.columns(table.full_table_name).map { |c| Column.new(c.name, c.sql_type, adapter.quote(c.default), c.null) }
-      end
+      connection.query(<<~SQL, 'COLUMN').map { |name, sql_type, default, nullable| Column.new(name, sql_type, adapter.quote(default), nullable || false) }
+        SELECT column_name, data_type, column_default, is_nullable FROM svv_columns WHERE table_schema = '#{table.schema_name}' and table_name = '#{table.table_name}';
+      SQL
     rescue ActiveRecord::ActiveRecordError, Mysql2::Error, PG::Error => e
       raise DataSource::ConnectionBad.new(e)
     end
 
     def fetch_rows(table, limit)
-      return [] if spectrum?(table)
+      return [] if spectrum?(table) || late_binding_view?(table)
       super
     end
 
     def fetch_count(table)
-      return 0 if spectrum?(table)
+      return 0 if spectrum?(table) || late_binding_view?(table)
+      super
+    end
+
+    def fetch_view_query(table)
+      return nil unless view?(table)
+      adapter = connection.pool.connection
+      connection.query(<<~SQL, 'VIEW QUERY').join("\n")
+        SELECT definition FROM pg_views WHERE schemaname = '#{table.schema_name}' and viewname = '#{table.table_name}';
+      SQL
+    end
+
+    def fetch_view_query_plan(query)
+      return nil if query.blank?
+      query = query.sub(/create view .*?as/, '').sub('with no schema binding', '')
       super
     end
 
@@ -61,5 +77,19 @@ def spectrum?(table)
       raise "@external_table_names must be defined, execute fetch_table befor spectrum?" unless instance_variable_defined?(:@external_table_names)
       @external_table_names.include?(table.full_table_name.split('.'))
     end
+
+    def view?(table)
+      base_view?(table) || late_binding_view?(table)
+    end
+
+    def base_view?(table)
+      raise "@view_names must be defined, execute fetch_table befor view?" unless instance_variable_defined?(:@view_names)
+      @view_names.include?(table.full_table_name.split('.'))
+    end
+
+    def late_binding_view?(table)
+      raise "@late_binding_view_names must be defined, execute fetch_table befor late_binding_view?" unless instance_variable_defined?(:@late_binding_view_names)
+      @late_binding_view_names.include?(table.full_table_name.split('.'))
+    end
   end
 end

app/models/data_source_adapters/standard_adapter.rb

@@ -33,6 +33,13 @@ def fetch_count(table)
       raise DataSource::ConnectionBad.new(e)
     end
 
+    def fetch_view_query_plan(query)
+      adapter = connection.pool.connection
+      connection.query("EXPLAIN #{query}", 'EXPLAIN').join("\n")
+    rescue ActiveRecord::ActiveRecordError, Mysql2::Error, PG::Error => e
+      raise DataSource::ConnectionBad.new(e)
+    end
+
     def source_base_class
       return DynamicTable.const_get(source_base_class_name) if DynamicTable.const_defined?(source_base_class_name)
 

app/models/data_source_table.rb

@@ -22,4 +22,12 @@ def fetch_rows(limit=20)
   def fetch_count
     @data_source.access_logging { data_source_adapter.fetch_count(self) }
   end
+
+  def fetch_view_query
+    @view_query ||= @data_source.access_logging { data_source_adapter.fetch_view_query(self) }
+  end
+
+  def fetch_view_query_plan
+    @view_query_plan ||= @data_source.access_logging { data_source_adapter.fetch_view_query_plan(@view_query) }
+  end
 end

app/models/table_memo.rb

@@ -7,6 +7,7 @@ class TableMemo < ApplicationRecord
   belongs_to :schema_memo
 
   has_one :raw_dataset, class_name: "TableMemoRawDataset", dependent: :destroy
+  has_one :view_meta_data, class_name: "ViewMetaDatum", dependent: :destroy
 
   has_many :column_memos, dependent: :destroy
   has_many :logs, -> { order(:id) }, class_name: "TableMemoLog", dependent: :destroy

app/models/view_meta_datum.rb

@@ -0,0 +1,3 @@
+class ViewMetaDatum < ApplicationRecord
+  belongs_to :table_memo
+end

app/views/table_memos/_raw_dataset.html.haml

@@ -0,0 +1,29 @@
+.box.data-box
+  .box-header
+    %h3.box-title Data
+  .box-body
+    - if @raw_dataset
+      %table.table.table-hover.table-bordered.table-striped.text-sm{ role: "grid" }
+        %tr
+          - @raw_dataset_columns.each do |column|
+            %th #{column.name} (#{column.sql_type})
+        - if @table_memo.masked?
+          %tr
+            %td.text-center{ colspan: @raw_dataset_columns.size }
+              = t("masked_table")
+        - else
+          - database_name = @table_memo.database_memo.name
+          - table_name = @table_memo.name
+          - @raw_dataset_rows.each do |row|
+            %tr
+              - Array.wrap(row).each_with_index do |value, i|
+                - column = @raw_dataset_columns[i]
+                %td
+                  - if MaskedDatum.masked_column?(database_name, table_name, column.name)
+                    = t("masked_text")
+                  - else
+                    = value
+      .pull-right.block
+        #{@raw_dataset_rows.try(:size).to_i} / #{@raw_dataset.count || '?' } records
+    - else
+      = t("no_preview_dataset")

app/views/table_memos/_view_meta_data.html.haml

@@ -0,0 +1,14 @@
+.box.data-box
+  .box-header
+    %h3.box-title View Query
+  .box-body
+    %pre
+      %code= @view_meta_data.query
+
+.box.data-box
+  .box-header
+    %h3.box-title View Query Plan
+  .box-body
+    %pre
+      %code= @view_meta_data.explain
+

app/views/table_memos/show.html.haml

@@ -60,32 +60,7 @@
         %th Nullable
       = render partial: "column_memo", collection: @table_memo.column_memos.sort_by(&:display_order)
 
-.box.data-box
-  .box-header
-    %h3.box-title Data
-  .box-body
-    - if @raw_dataset
-      %table.table.table-hover.table-bordered.table-striped.text-sm{ role: "grid" }
-        %tr
-          - @raw_dataset_columns.each do |column|
-            %th #{column.name} (#{column.sql_type})
-        - if @table_memo.masked?
-          %tr
-            %td.text-center{ colspan: @raw_dataset_columns.size }
-              = t("masked_table")
-        - else
-          - database_name = @table_memo.database_memo.name
-          - table_name = @table_memo.name
-          - @raw_dataset_rows.each do |row|
-            %tr
-              - Array.wrap(row).each_with_index do |value, i|
-                - column = @raw_dataset_columns[i]
-                %td
-                  - if MaskedDatum.masked_column?(database_name, table_name, column.name)
-                    = t("masked_text")
-                  - else
-                    = value
-      .pull-right.block
-        #{@raw_dataset_rows.try(:size).to_i} / #{@raw_dataset.count || '?' } records
-    - else
-      = t("no_preview_dataset")
+- if @view_meta_data
+  = render partial: "view_meta_data"
+- else
+  = render partial: "raw_dataset"

db/view_meta_data.schema

@@ -0,0 +1,6 @@
+create_table "view_meta_data", force: :cascade do |t|
+  t.integer "table_memo_id",              null: false
+  t.text    "query",                      null: false
+  t.text    "explain",                    null: false
+  t.datetime "created_at",                null: false
+end

spec/models/data_source_adapters/mysql2_adapter_spec.rb

@@ -26,6 +26,7 @@ def execute_sql(sql)
       CREATE TABLE IF NOT EXISTS #{table.table_name} (id INT PRIMARY KEY);
       TRUNCATE TABLE #{table.table_name};
       INSERT INTO #{table.table_name} VALUES (1);
+      ANALYZE TABLE #{table.table_name};
     SQL
   end