Merge pull request #353 from Charcoal-SE/sql-explorer

SQL data exploder

.gitignore

@@ -18,6 +18,7 @@
 config/secrets.html
 config/config.yml
 config/database.yml
+config/blazer.yml
 
 Gemfile.lock    # we need the Gemfile, but not the .lock
 

Gemfile

@@ -15,7 +15,7 @@ gem 'devise', git: 'https://github.com/plataformatec/devise'
 gem 'htmlentities', '~> 4.3', '>= 4.3.4'
 gem 'httparty'
 gem 'jwt'
-gem 'loofah'
+gem 'loofah', '~> 2.2.1'
 gem 'mysql2'
 gem 'octokit', '~> 4.0'
 gem 'puma'
@@ -41,6 +41,7 @@ gem 'groupdate', '~> 3.0.0'
 gem 'rubocop', '~> 0.48.1', require: false, group: :test
 gem 'simplecov', require: false, group: :test
 
+gem 'blazer', '1.8.2'
 gem 'grape', '1.0.0'
 
 # gem 'puma_worker_killer'

Gemfile.lock

@@ -100,6 +100,11 @@ GEM
     backports (3.11.0)
     bcrypt (3.1.11)
     bindex (0.5.0)
+    blazer (1.8.2)
+      activerecord (>= 4)
+      chartkick
+      railties (>= 4)
+      safely_block (>= 0.1.1)
     builder (3.2.3)
     byebug (9.1.0)
     callsite (0.0.11)
@@ -144,6 +149,7 @@ GEM
       thread_safe (~> 0.3, >= 0.3.1)
     docile (1.1.5)
     equalizer (0.0.11)
+    errbase (0.1.0)
     erubi (1.7.0)
     ethon (0.11.0)
       ffi (>= 1.3.0)
@@ -193,7 +199,7 @@ GEM
     jwt (2.1.0)
     launchy (2.4.3)
       addressable (~> 2.3)
-    loofah (2.2.0)
+    loofah (2.2.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
@@ -295,6 +301,8 @@ GEM
       unicode-display_width (~> 1.0, >= 1.0.1)
     ruby-progressbar (1.9.0)
     safe_yaml (1.0.4)
+    safely_block (0.2.1)
+      errbase
     sass (3.5.4)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
@@ -384,6 +392,7 @@ DEPENDENCIES
   aescrypt (~> 2.0, >= 2.0.2)!
   audited (~> 4.4)
   awesome_print
+  blazer (= 1.8.2)
   byebug
   capistrano
   capistrano-bundler
@@ -406,7 +415,7 @@ DEPENDENCIES
   jbuilder (~> 2.0)
   jquery-rails
   jwt
-  loofah
+  loofah (~> 2.2.1)
   meta_request
   mysql2
   octokit (~> 4.0)

config/blazer.sample.yml

@@ -0,0 +1,27 @@
+# see https://github.com/ankane/blazer for more info
+
+data_sources:
+  main:
+    url: mysql2://metasmoke_blazer:zFpc8tw7CdAuXizX@localhost:3306/metasmoke_dev
+    timeout: 10
+    cache:
+      mode: slow
+      expires_in: 60
+      slow_threshold: 5
+
+    smart_variables:
+      reason_id: "SELECT id, reason_name FROM p_reasons ORDER BY id ASC"
+      user_id: "SELECT id, username FROM p_users ORDER BY id ASC"
+      api_key_id: "SELECT id, app_name FROM p_api_keys ORDER BY id ASC"
+      site_id: "SELECT id, site_name FROM p_sites ORDER BY id ASC"
+
+    linked_columns:
+      # user_id: "/admin/users/{value}"
+
+    smart_columns:
+      reason_id: "SELECT id, reason_name FROM p_reasons WHERE id IN {value}"
+      user_id: "SELECT id, username FROM p_users WHERE id IN {value}"
+      api_key_id: "SELECT id, app_name FROM p_api_keys WHERE id IN {value}"
+      site_id: "SELECT id, site_name FROM p_sites WHERE id IN {value}"
+
+audit: true

config/routes.rb

@@ -270,6 +270,10 @@
     root             to: 'data#index',        as: :data_explorer
     get  'retrieve', to: 'data#retrieve',     as: :data_retrieve
     get  'schema',   to: 'data#table_schema', as: :data_schema
+
+    authenticate(:user, ->(user) { user.has_role?(:core) }) do
+      mount Blazer::Engine, at: 'sql'
+    end
   end
 
   scope '/domains' do

db/migrate/20180321002436_run_create_views.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class RunCreateViews < ActiveRecord::Migration[5.2]
+  def change
+    success = system 'rails runner db/scripts/create_views.rb'
+    raise StandardError, 'View script had non-zero exit code' unless success
+  end
+end

db/migrate/20180321004613_install_blazer.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class InstallBlazer < ActiveRecord::Migration[5.2]
+  def change
+    create_table :blazer_queries do |t|
+      t.references :creator
+      t.string :name
+      t.text :description
+      t.text :statement
+      t.string :data_source
+      t.timestamps null: false
+    end
+
+    create_table :blazer_audits do |t|
+      t.references :user
+      t.references :query
+      t.text :statement
+      t.string :data_source
+      t.timestamp :created_at
+    end
+
+    create_table :blazer_dashboards do |t|
+      t.references :creator
+      t.text :name
+      t.timestamps null: false
+    end
+
+    create_table :blazer_dashboard_queries do |t|
+      t.references :dashboard
+      t.references :query
+      t.integer :position
+      t.timestamps null: false
+    end
+
+    create_table :blazer_checks do |t|
+      t.references :creator
+      t.references :query
+      t.string :state
+      t.string :schedule
+      t.text :emails
+      t.string :check_type
+      t.text :message
+      t.timestamp :last_run_at
+      t.timestamps null: false
+    end
+  end
+end

db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2018_03_08_104133) do
+ActiveRecord::Schema.define(version: 2018_03_21_004613) do
 
   create_table "announcements", options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
     t.text "text"
@@ -64,6 +64,60 @@
     t.index ["user_id", "user_type"], name: "user_index", length: { user_type: 191 }
   end
 
+  create_table "blazer_audits", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.bigint "user_id"
+    t.bigint "query_id"
+    t.text "statement"
+    t.string "data_source"
+    t.timestamp "created_at"
+    t.index ["query_id"], name: "index_blazer_audits_on_query_id"
+    t.index ["user_id"], name: "index_blazer_audits_on_user_id"
+  end
+
+  create_table "blazer_checks", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.bigint "creator_id"
+    t.bigint "query_id"
+    t.string "state"
+    t.string "schedule"
+    t.text "emails"
+    t.string "check_type"
+    t.text "message"
+    t.timestamp "last_run_at"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["creator_id"], name: "index_blazer_checks_on_creator_id"
+    t.index ["query_id"], name: "index_blazer_checks_on_query_id"
+  end
+
+  create_table "blazer_dashboard_queries", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.bigint "dashboard_id"
+    t.bigint "query_id"
+    t.integer "position"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["dashboard_id"], name: "index_blazer_dashboard_queries_on_dashboard_id"
+    t.index ["query_id"], name: "index_blazer_dashboard_queries_on_query_id"
+  end
+
+  create_table "blazer_dashboards", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.bigint "creator_id"
+    t.text "name"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["creator_id"], name: "index_blazer_dashboards_on_creator_id"
+  end
+
+  create_table "blazer_queries", options: "ENGINE=InnoDB DEFAULT CHARSET=utf8", force: :cascade do |t|
+    t.bigint "creator_id"
+    t.string "name"
+    t.text "description"
+    t.text "statement"
+    t.string "data_source"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["creator_id"], name: "index_blazer_queries_on_creator_id"
+  end
+
   create_table "commit_statuses", id: :integer, options: "ENGINE=InnoDB DEFAULT CHARSET=latin1", force: :cascade do |t|
     t.string "commit_sha", collation: "utf8_unicode_ci"
     t.string "status", collation: "utf8_unicode_ci"

db/scripts/create_views.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+EXCLUDE_TABLES = %w[ar_internal_metadata flags github_tokens schema_migrations].freeze
+EXCLUDE_COLUMNS = {
+  'api_keys' => ['key'],
+  'api_tokens' => %w[code token],
+  'audits' => ['remote_address'],
+  'smoke_detectors' => ['access_token'],
+  'users' => %w[email encrypted_password reset_password_token encrypted_api_token two_factor_token enabled_2fa salt iv]
+}.freeze
+
+tables = ActiveRecord::Base.connection.tables
+queries = []
+
+queries << "CREATE USER IF NOT EXISTS metasmoke_blazer@localhost IDENTIFIED BY 'zFpc8tw7CdAuXizX';"
+
+tables.each do |t|
+  next if EXCLUDE_TABLES.include? t
+  columns = ActiveRecord::Base.connection.columns(t).map(&:name) - (EXCLUDE_COLUMNS[t] || [])
+  queries << "CREATE VIEW p_#{t} AS SELECT #{columns.join(', ')} FROM #{t};"
+  queries << "GRANT SELECT ON #{ActiveRecord::Base.connection.current_database}.p_#{t} TO metasmoke_blazer@localhost;"
+end
+
+queries.each do |q|
+  ActiveRecord::Base.connection.execute q
+end