From ec6569b67d6f97a1c620f5b5849c1bd07be4ebe8 Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Wed, 16 Aug 2023 17:25:03 -0400 Subject: [PATCH 1/2] Log the tables sizes and statistics with db activity It's helpful to log the tables sorted by row counts and other information such as the autovacuum/analzye times. Even if we truncate the line to 8k (default for the logger), the largest row tables should be represented. --- app/models/vmdb_database_connection.rb | 43 ++++++--- spec/models/vmdb_database_connection_spec.rb | 95 ++++++++++++++++++-- 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/app/models/vmdb_database_connection.rb b/app/models/vmdb_database_connection.rb index e3c717d9a44..ae1f57dd5cd 100644 --- a/app/models/vmdb_database_connection.rb +++ b/app/models/vmdb_database_connection.rb @@ -25,25 +25,40 @@ class VmdbDatabaseConnection < ApplicationRecord virtual_column :blocked_by, :type => :integer def self.log_statistics(output = $log) + log_activity(output) + log_table_size(output) + log_table_statistics(output) + end + + def self.log_csv(keys, stats, label, output) require 'csv' + csv = CSV.generate do |rows| + rows << keys + stats.each { |s| rows << s.values_at(*keys) } + end - begin - stats = all.map(&:to_csv_hash) + output.info("MIQ(#{name}.#{__method__}) <<-#{label}\n#{csv}#{label}") + end - keys = stats.first.keys + def self.log_activity(output = $log) + stats = all.map(&:to_csv_hash) + log_csv(stats.first.keys, stats, "ACTIVITY_STATS_CSV", output) + rescue => err + output.warn("MIQ(#{name}.#{__method__}) Unable to log activity, '#{err.message}'") + end - csv = CSV.generate do |rows| - rows << keys - stats.each do |s| - vals = s.values_at(*keys) - rows << vals - end - end + def self.log_table_size(output = $log) + stats = ApplicationRecord.connection.table_size + log_csv(stats.first.keys, stats, "TABLE_SIZE_CSV", output) + rescue => err + output.warn("MIQ(#{name}.#{__method__}) Unable to log activity, '#{err.message}'") + end - output.info("MIQ(#{name}.#{__method__}) <<-ACTIVITY_STATS_CSV\n#{csv}ACTIVITY_STATS_CSV") - rescue => err - output.warn("MIQ(#{name}.#{__method__}) Unable to log stats, '#{err.message}'") - end + def self.log_table_statistics(output = $log) + stats = ApplicationRecord.connection.table_statistics.sort_by { |h| h['rows_live'] }.reverse! + log_csv(stats.first.keys, stats, "TABLE_STATS_CSV", output) + rescue => err + output.warn("MIQ(#{name}.#{__method__}) Unable to log activity, '#{err.message}'") end def address diff --git a/spec/models/vmdb_database_connection_spec.rb b/spec/models/vmdb_database_connection_spec.rb index 7e58d830c10..5fd2c65a7a4 100644 --- a/spec/models/vmdb_database_connection_spec.rb +++ b/spec/models/vmdb_database_connection_spec.rb @@ -113,7 +113,7 @@ end end - describe ".log_statistics" do + describe ".log_activity" do before do @buffer = StringIO.new class << @buffer @@ -123,9 +123,9 @@ class << @buffer end it "normal" do - described_class.log_statistics(@buffer) + described_class.log_activity(@buffer) lines = @buffer.string.lines - expect(lines.shift).to eq "MIQ(VmdbDatabaseConnection.log_statistics) <<-ACTIVITY_STATS_CSV\n" + expect(lines.shift).to eq "MIQ(VmdbDatabaseConnection.log_csv) <<-ACTIVITY_STATS_CSV\n" expect(lines.pop).to eq "ACTIVITY_STATS_CSV" header, *rows = CSV.parse lines.join @@ -154,8 +154,93 @@ class << @buffer it "exception" do allow(described_class).to receive(:all).and_raise("FAILURE") - described_class.log_statistics(@buffer) - expect(@buffer.string.lines.first).to eq("MIQ(VmdbDatabaseConnection.log_statistics) Unable to log stats, 'FAILURE'") + described_class.log_activity(@buffer) + expect(@buffer.string.lines.first).to eq("MIQ(VmdbDatabaseConnection.log_activity) Unable to log activity, 'FAILURE'") + end + end + + + describe ".log_table_size" do + before do + @buffer = StringIO.new + class << @buffer + alias_method :info, :write + alias_method :warn, :write + end + end + + it "normal" do + described_class.log_table_size(@buffer) + lines = @buffer.string.lines + expect(lines.shift).to eq "MIQ(VmdbDatabaseConnection.log_csv) <<-TABLE_SIZE_CSV\n" + expect(lines.pop).to eq "TABLE_SIZE_CSV" + + header, *rows = CSV.parse lines.join + expect(header).to eq %w( + table_name + rows + pages + size + average_row_size + ) + + expect(rows.length).to be > 0 + rows.each do |row| + expect(row.first).to be_truthy + end + end + + it "exception" do + allow(ApplicationRecord.connection).to receive(:table_size).and_raise("FAILURE") + described_class.log_table_size(@buffer) + expect(@buffer.string.lines.first).to eq("MIQ(VmdbDatabaseConnection.log_table_size) Unable to log activity, 'FAILURE'") + end + end + + describe ".log_table_statistics" do + before do + @buffer = StringIO.new + class << @buffer + alias_method :info, :write + alias_method :warn, :write + end + end + + it "normal" do + described_class.log_table_statistics(@buffer) + lines = @buffer.string.lines + expect(lines.shift).to eq "MIQ(VmdbDatabaseConnection.log_csv) <<-TABLE_STATS_CSV\n" + expect(lines.pop).to eq "TABLE_STATS_CSV" + + header, *rows = CSV.parse lines.join + expect(header).to eq %w( + table_name + table_scans + sequential_rows_read + index_scans + index_rows_fetched + rows_inserted + rows_updated + rows_deleted + rows_hot_updated + rows_live + rows_dead + last_vacuum_date + last_autovacuum_date + last_analyze_date + last_autoanalyze_date + ) + + expect(rows.length).to be > 0 + rows.each do |row| + expect(row.first).to be_truthy + end + end + + it "exception" do + allow(ApplicationRecord.connection).to receive(:table_statistics).and_raise("FAILURE") + described_class.log_table_statistics(@buffer) + expect(@buffer.string.lines.first).to eq("MIQ(VmdbDatabaseConnection.log_table_statistics) Unable to log activity, 'FAILURE'") end end end From 9bd5b139c29f86e7838e94d3a2c46fcda0b265f9 Mon Sep 17 00:00:00 2001 From: Joe Rafaniello Date: Thu, 7 Sep 2023 17:51:35 -0400 Subject: [PATCH 2/2] Eliminate non-regular tables/non-user tables --- app/models/vmdb_database_connection.rb | 2 +- lib/extensions/ar_adapter/ar_dba.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/models/vmdb_database_connection.rb b/app/models/vmdb_database_connection.rb index ae1f57dd5cd..019e3653274 100644 --- a/app/models/vmdb_database_connection.rb +++ b/app/models/vmdb_database_connection.rb @@ -55,7 +55,7 @@ def self.log_table_size(output = $log) end def self.log_table_statistics(output = $log) - stats = ApplicationRecord.connection.table_statistics.sort_by { |h| h['rows_live'] }.reverse! + stats = ApplicationRecord.connection.table_statistics log_csv(stats.first.keys, stats, "TABLE_STATS_CSV", output) rescue => err output.warn("MIQ(#{name}.#{__method__}) Unable to log activity, '#{err.message}'") diff --git a/lib/extensions/ar_adapter/ar_dba.rb b/lib/extensions/ar_adapter/ar_dba.rb index a35e95ff8cd..dc3415d9945 100644 --- a/lib/extensions/ar_adapter/ar_dba.rb +++ b/lib/extensions/ar_adapter/ar_dba.rb @@ -401,7 +401,7 @@ def table_statistics , last_autovacuum AS last_autovacuum_date , last_analyze AS last_analyze_date , last_autoanalyze AS last_autoanalyze_date - FROM pg_stat_all_tables + FROM pg_stat_user_tables WHERE schemaname NOT IN ('pg_catalog', 'information_schema') ORDER BY relname ASC ; SQL @@ -462,6 +462,7 @@ def table_size FROM pg_class WHERE reltuples > 1 AND relname NOT LIKE 'pg_%' + AND relkind = 'r' ORDER BY reltuples DESC , relpages DESC ; SQL