diff --git a/alembic/versions/1e188109fc6b_bug_1398200_1457484_remove_plugins.py b/alembic/versions/1e188109fc6b_bug_1398200_1457484_remove_plugins.py new file mode 100644 index 0000000000..f03cb0eb38 --- /dev/null +++ b/alembic/versions/1e188109fc6b_bug_1398200_1457484_remove_plugins.py @@ -0,0 +1,44 @@ +"""bug 1398200, 1457484 remove unused tables + +Revision ID: 1e188109fc6b +Revises: 8e8390138426 +Create Date: 2018-04-27 14:12:16.709146 + +""" + +from alembic import op + +from socorro.lib.migrations import load_stored_proc + + +# revision identifiers, used by Alembic. +revision = '1e188109fc6b' +down_revision = '8e8390138426' + + +def upgrade(): + op.execute('DROP TABLE IF EXISTS plugins') + op.execute('DROP TABLE IF EXISTS release_channel_matches') + op.execute('DROP TABLE IF EXISTS replication_test') + op.execute('DROP TABLE IF EXISTS sessions') + op.execute('DROP TABLE IF EXISTS socorro_db_version') + op.execute('DROP TABLE IF EXISTS socorro_db_version_history') + op.execute('DROP TABLE IF EXISTS transform_rules') + op.execute('DROP TABLE IF EXISTS crashes_by_user') + op.execute('DROP TABLE IF EXISTS crashes_by_user_build') + op.execute('DROP TABLE IF EXISTS uptime_levels') + op.execute('DROP TABLE IF EXISTS modules') + op.execute('DROP TABLE IF EXISTS crash_types') + op.execute('DROP TABLE IF EXISTS process_types') + op.execute('DROP TABLE IF EXISTS rank_compare') + + op.execute('DROP FUNCTION IF EXISTS backfill_rank_compare(date)') + op.execute('DROP FUNCTION IF EXISTS update_rank_compare(date, boolean, interval)') + + # Load the new version of backfill_matviews + load_stored_proc(op, ['backfill_matviews.sql']) + + +def downgrade(): + # There is no going back + pass diff --git a/socorro/external/postgresql/crashstorage.py b/socorro/external/postgresql/crashstorage.py index 71f1075062..74b743979e 100644 --- a/socorro/external/postgresql/crashstorage.py +++ b/socorro/external/postgresql/crashstorage.py @@ -103,8 +103,7 @@ def save_processed(self, processed_crash): self.transaction(self._save_processed_transaction, processed_crash) def _save_processed_transaction(self, connection, processed_crash): - report_id = self._save_processed_report(connection, processed_crash) - self._save_plugins(connection, processed_crash, report_id) + self._save_processed_report(connection, processed_crash) def _save_processed_report(self, connection, processed_crash): """Here we INSERT or UPDATE a row in the reports table. @@ -199,48 +198,6 @@ def print_as(a, b): report_id = single_value_sql(connection, upsert_sql, value_list) return report_id - def _save_plugins(self, connection, processed_crash, report_id): - """ Electrolysis Support - Optional - processed_crash may contain a - ProcessType of plugin. In the future this value would be default, - content, maybe even Jetpack... This indicates which process was the - crashing process. - plugin - When set to plugin, the jsonDocument MUST calso contain - PluginFilename and PluginName - """ - process_type = processed_crash['process_type'] - if not process_type: - return - - if process_type == "plugin": - - # Bug#543776 We actually will are relaxing the non-null policy... - # a null filename, name, and version is OK. We'll use empty strings - try: - plugin_filename = processed_crash['PluginFilename'] - plugin_name = processed_crash['PluginName'] - except KeyError as x: - self.config.logger.error( - 'the crash is missing a required field: %s', str(x) - ) - return - find_plugin_sql = ('select id from plugins ' - 'where filename = %s ' - 'and name = %s') - try: - single_value_sql( - connection, - find_plugin_sql, - (plugin_filename, plugin_name) - ) - except SQLDidNotReturnSingleValue: - insert_plugsins_sql = ("insert into plugins (filename, name) " - "values (%s, %s) returning id") - execute_no_results( - connection, - insert_plugsins_sql, - (plugin_filename, plugin_name) - ) - @staticmethod def _table_suffix_for_crash_id(crash_id): """given an crash_id, return the name of its storage table""" diff --git a/socorro/external/postgresql/models.py b/socorro/external/postgresql/models.py index 1dd7080294..3d9ed64da3 100644 --- a/socorro/external/postgresql/models.py +++ b/socorro/external/postgresql/models.py @@ -9,7 +9,6 @@ from __future__ import unicode_literals from sqlalchemy import Column, ForeignKey, Index, text, Integer -from sqlalchemy import event from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship import sqlalchemy.types as types @@ -27,7 +26,6 @@ ARRAY, BIGINT, SMALLINT, - CHAR, ) from sqlalchemy.dialects.postgresql.base import ischema_names @@ -217,16 +215,6 @@ def __repr__(self): ############################### -class Module(DeclarativeBase): - __tablename__ = 'modules' - - # column definitions - module_id = Column(u'module_id', INTEGER(), - nullable=False, primary_key=True) - name = Column(u'name', TEXT(), nullable=False, primary_key=True) - version = Column(u'version', TEXT(), nullable=False, primary_key=True) - - class RawAdiLogs(DeclarativeBase): __tablename__ = 'raw_adi_logs' @@ -282,16 +270,6 @@ class RawAdi(DeclarativeBase): ) -class ReplicationTest(DeclarativeBase): - __tablename__ = 'replication_test' - - # column definitions - id = Column(u'id', SMALLINT()) - test = Column(u'test', BOOLEAN()) - - __mapper_args__ = {"primary_key": (id, test)} - - class ReportsBad(DeclarativeBase): __tablename__ = 'reports_bad' uuid = Column(u'uuid', TEXT(), nullable=False) @@ -410,70 +388,6 @@ class BuildAdu(DeclarativeBase): ) -class CrashType(DeclarativeBase): - __tablename__ = 'crash_types' - - # column definitions - crash_type = Column(u'crash_type', CITEXT(), - nullable=False, index=True, unique=True) - crash_type_id = Column(u'crash_type_id', INTEGER(), - primary_key=True, nullable=False) - crash_type_short = Column(u'crash_type_short', CITEXT(), - nullable=False, index=True, unique=True) - has_hang_id = Column(u'has_hang_id', BOOLEAN()) - include_agg = Column(u'include_agg', BOOLEAN(), - nullable=False, server_default=text('True')) - old_code = Column(u'old_code', CHAR(length=1), nullable=False) - process_type = Column(u'process_type', CITEXT(), ForeignKey( - 'process_types.process_type'), nullable=False) - - # relationship definitions - process_types = relationship( - 'ProcessType', primaryjoin='CrashType.process_type==ProcessType.process_type') - - -class CrashesByUser(DeclarativeBase): - __tablename__ = 'crashes_by_user' - - # column definitions - adu = Column(u'adu', INTEGER(), nullable=False) - crash_type_id = Column(u'crash_type_id', INTEGER(), ForeignKey( - 'crash_types.crash_type_id'), primary_key=True, nullable=False) - os_short_name = Column(u'os_short_name', CITEXT(), - primary_key=True, nullable=False) - product_version_id = Column(u'product_version_id', INTEGER(), - primary_key=True, nullable=False, autoincrement=False) - report_count = Column(u'report_count', INTEGER(), nullable=False) - report_date = Column(u'report_date', DATE(), - primary_key=True, nullable=False) - - # relationship definitions - crash_types = relationship( - 'CrashType', primaryjoin='CrashesByUser.crash_type_id==CrashType.crash_type_id') - - -class CrashesByUserBuild(DeclarativeBase): - __tablename__ = 'crashes_by_user_build' - - # column definitions - adu = Column(u'adu', INTEGER(), nullable=False) - build_date = Column(u'build_date', DATE(), - primary_key=True, nullable=False) - crash_type_id = Column(u'crash_type_id', INTEGER(), ForeignKey( - 'crash_types.crash_type_id'), primary_key=True, nullable=False) - os_short_name = Column(u'os_short_name', CITEXT(), - primary_key=True, nullable=False) - product_version_id = Column(u'product_version_id', INTEGER(), primary_key=True, - nullable=False, autoincrement=False) - report_count = Column(u'report_count', INTEGER(), nullable=False) - report_date = Column(u'report_date', DATE(), - primary_key=True, nullable=False) - - # relationship definitions - crash_types = relationship( - 'CrashType', primaryjoin='CrashesByUserBuild.crash_type_id==CrashType.crash_type_id') - - class Domain(DeclarativeBase): __tablename__ = 'domains' @@ -539,27 +453,6 @@ class OsVersion(DeclarativeBase): 'OsName', primaryjoin='OsVersion.os_name==OsName.os_name') -class Plugin(DeclarativeBase): - __tablename__ = 'plugins' - - # column definitions - filename = Column(u'filename', TEXT(), nullable=False) - id = Column(u'id', INTEGER(), primary_key=True, nullable=False) - name = Column(u'name', TEXT(), nullable=False) - - __table_args__ = ( - Index('filename_name_key', filename, name, unique=True), - ) - - -class ProcessType(DeclarativeBase): - __tablename__ = 'process_types' - - # column definitions - process_type = Column(u'process_type', CITEXT(), - primary_key=True, nullable=False) - - class Product(DeclarativeBase): __tablename__ = 'products' @@ -737,28 +630,6 @@ class ProductVersionBuild(DeclarativeBase): ) -class RankCompare(DeclarativeBase): - __tablename__ = 'rank_compare' - - # column definitions - percent_of_total = Column(u'percent_of_total', NUMERIC()) - product_version_id = Column(u'product_version_id', INTEGER(), - primary_key=True, nullable=False, autoincrement=False) - rank_days = Column(u'rank_days', INTEGER(), - primary_key=True, nullable=False) - rank_report_count = Column(u'rank_report_count', INTEGER()) - report_count = Column(u'report_count', INTEGER()) - signature_id = Column(u'signature_id', INTEGER(), - primary_key=True, nullable=False, index=True) - total_reports = Column(u'total_reports', BIGINT()) - - # Indexes - __table_args__ = ( - Index('rank_compare_product_version_id_rank_report_count', - product_version_id, rank_report_count), - ) - - class RawCrashes(DeclarativeBase): __tablename__ = 'raw_crashes' @@ -803,24 +674,6 @@ class ReleaseChannel(DeclarativeBase): ) -# DEPRECATED -> enum that's translated to a table for -# 001_update_reports_clean.sql -class ReleaseChannelMatch(DeclarativeBase): - __tablename__ = 'release_channel_matches' - - # column definitions - match_string = Column(u'match_string', TEXT(), - primary_key=True, nullable=False) - release_channel = Column(u'release_channel', CITEXT(), ForeignKey( - 'release_channels.release_channel'), primary_key=True, nullable=False) - - # relationship definitions - release_channels = relationship( - 'ReleaseChannel', - primaryjoin='ReleaseChannelMatch.release_channel==ReleaseChannel.release_channel' - ) - - class ReleaseRepository(DeclarativeBase): __tablename__ = 'release_repositories' @@ -942,16 +795,6 @@ class ReportsUserInfo(DeclarativeBase): uuid = Column(u'uuid', TEXT(), primary_key=True, nullable=False) -class Session(DeclarativeBase): - __tablename__ = 'sessions' - - # column definitions - data = Column(u'data', TEXT(), nullable=False) - last_activity = Column(u'last_activity', INTEGER(), nullable=False) - session_id = Column(u'session_id', VARCHAR(length=127), - primary_key=True, nullable=False) - - class Signature(DeclarativeBase): __tablename__ = 'signatures' @@ -1026,25 +869,6 @@ class GraphicsDevice(DeclarativeBase): adapter_name = Column(u'adapter_name', TEXT()) -class SocorroDbVersion(DeclarativeBase): - __tablename__ = 'socorro_db_version' - - # column definitions - current_version = Column(u'current_version', TEXT(), - primary_key=True, nullable=False) - refreshed_at = Column(u'refreshed_at', TIMESTAMP(timezone=True)) - - -class SocorroDbVersionHistory(DeclarativeBase): - __tablename__ = 'socorro_db_version_history' - - # column definitions - backfill_to = Column(u'backfill_to', DATE()) - upgraded_on = Column(u'upgraded_on', TIMESTAMP( - timezone=True), primary_key=True, nullable=False, server_default=text('NOW()')) - version = Column(u'version', TEXT(), primary_key=True, nullable=False) - - class SpecialProductPlatform(DeclarativeBase): """ Currently used for android platform. Uses platform, product name, repo, build_type to rename a product_name """ @@ -1066,42 +890,6 @@ class SpecialProductPlatform(DeclarativeBase): # build_type = Column(u'build_type', build_type(), primary_key=True, nullable=False) -class TransformRule(DeclarativeBase): - __tablename__ = 'transform_rules' - - # column definitions - transform_rule_id = Column( - u'transform_rule_id', INTEGER(), primary_key=True, nullable=False) - category = Column(u'category', CITEXT(), nullable=False) - rule_order = Column(u'rule_order', INTEGER(), nullable=False) - action = Column(u'action', TEXT(), nullable=False, server_default='') - action_args = Column(u'action_args', TEXT(), - nullable=False, server_default='') - action_kwargs = Column(u'action_kwargs', TEXT(), - nullable=False, server_default='') - predicate = Column(u'predicate', TEXT(), nullable=False, server_default='') - predicate_args = Column(u'predicate_args', TEXT(), - nullable=False, server_default='') - predicate_kwargs = Column( - u'predicate_kwargs', TEXT(), nullable=False, server_default='') - - __table_args__ = ( - Index('transform_rules_key', category, rule_order, unique=True), - ) - - -class UptimeLevel(DeclarativeBase): - __tablename__ = 'uptime_levels' - - # column definitions - max_uptime = Column(u'max_uptime', INTERVAL(), nullable=False) - min_uptime = Column(u'min_uptime', INTERVAL(), nullable=False) - uptime_level = Column(u'uptime_level', INTEGER(), - primary_key=True, nullable=False, autoincrement=False) - uptime_string = Column(u'uptime_string', CITEXT(), - nullable=False, index=True, unique=True) - - class Crontabber(DeclarativeBase): __tablename__ = 'crontabber' @@ -1219,19 +1007,3 @@ class Correlations(DeclarativeBase): unique=True ), ) - - -########################################### -# Schema definition: Aggregates -########################################### - -@event.listens_for(UptimeLevel.__table__, "after_create") -def array_accum(target, connection, **kw): - array_accum = """ - CREATE AGGREGATE array_accum(anyelement) ( - SFUNC = array_append, - STYPE = anyarray, - INITCOND = '{}' - ) - """ - connection.execute(array_accum) diff --git a/socorro/external/postgresql/raw_sql/procs/001_update_reports_clean.sql b/socorro/external/postgresql/raw_sql/procs/001_update_reports_clean.sql index 675de67d65..ab629e7ed6 100644 --- a/socorro/external/postgresql/raw_sql/procs/001_update_reports_clean.sql +++ b/socorro/external/postgresql/raw_sql/procs/001_update_reports_clean.sql @@ -412,5 +412,3 @@ RETURN TRUE; END; $_$; - - diff --git a/socorro/external/postgresql/raw_sql/procs/backfill_matviews.sql b/socorro/external/postgresql/raw_sql/procs/backfill_matviews.sql index 5789f38cb7..265348960b 100644 --- a/socorro/external/postgresql/raw_sql/procs/backfill_matviews.sql +++ b/socorro/external/postgresql/raw_sql/procs/backfill_matviews.sql @@ -77,9 +77,5 @@ WHILE thisday <= lastday LOOP END LOOP; --- finally rank_compare, which doesn't need to be filled in for each day -RAISE INFO 'rank_compare'; -PERFORM backfill_rank_compare(lastday); - RETURN true; END; $$; diff --git a/socorro/external/postgresql/raw_sql/procs/backfill_rank_compare.sql b/socorro/external/postgresql/raw_sql/procs/backfill_rank_compare.sql deleted file mode 100644 index 67de68fef9..0000000000 --- a/socorro/external/postgresql/raw_sql/procs/backfill_rank_compare.sql +++ /dev/null @@ -1,11 +0,0 @@ -CREATE OR REPLACE FUNCTION backfill_rank_compare(updateday date DEFAULT NULL::date) RETURNS boolean - LANGUAGE plpgsql - AS $$ -BEGIN - -PERFORM update_rank_compare(updateday, false); - -RETURN TRUE; -END; $$; - - diff --git a/socorro/external/postgresql/raw_sql/procs/update_rank_compare.sql b/socorro/external/postgresql/raw_sql/procs/update_rank_compare.sql deleted file mode 100644 index ad10af4488..0000000000 --- a/socorro/external/postgresql/raw_sql/procs/update_rank_compare.sql +++ /dev/null @@ -1,70 +0,0 @@ -CREATE OR REPLACE FUNCTION update_rank_compare(updateday date DEFAULT NULL::date, checkdata boolean DEFAULT true, check_period interval DEFAULT '01:00:00'::interval) RETURNS boolean - LANGUAGE plpgsql - SET client_min_messages TO 'ERROR' - AS $$ -BEGIN --- this function populates a daily matview --- for rankings of signatures on TCBS --- depends on the new reports_clean - --- run for yesterday if not set -updateday := COALESCE(updateday, ( CURRENT_DATE -1 )); - --- don't care if we've been run --- since there's no historical data - --- check if reports_clean is complete -IF NOT reports_clean_done(updateday, check_period) THEN - IF checkdata THEN - RAISE NOTICE 'Reports_clean has not been updated to the end of %',updateday; - RETURN FALSE; - ELSE - RETURN FALSE; - END IF; -END IF; - --- obtain a lock on the matview so that we can TRUNCATE -IF NOT try_lock_table('rank_compare', 'ACCESS EXCLUSIVE') THEN - RAISE NOTICE 'unable to lock the rank_compare table for update.'; - RETURN FALSE; -END IF; - --- create temporary table with totals from reports_clean - -CREATE TEMPORARY TABLE prod_sig_counts -AS SELECT product_version_id, signature_id, count(*) as report_count -FROM reports_clean -WHERE utc_day_is(date_processed, updateday) -GROUP BY product_version_id, signature_id; - --- truncate rank_compare since we don't need the old data - -TRUNCATE rank_compare CASCADE; - --- now insert the new records -INSERT INTO rank_compare ( - product_version_id, signature_id, - rank_days, - report_count, - total_reports, - rank_report_count, - percent_of_total) -SELECT product_version_id, signature_id, - 1, - report_count, - total_count, - count_rank, - round(( report_count::numeric / total_count ),5) -FROM ( - SELECT product_version_id, signature_id, - report_count, - sum(report_count) over (partition by product_version_id) as total_count, - dense_rank() over (partition by product_version_id - order by report_count desc) as count_rank - FROM prod_sig_counts -) as initrank; - -RETURN TRUE; -END; $$; - - diff --git a/socorro/external/postgresql/staticdata.py b/socorro/external/postgresql/staticdata.py index e25d12f93f..feca273fe2 100644 --- a/socorro/external/postgresql/staticdata.py +++ b/socorro/external/postgresql/staticdata.py @@ -29,14 +29,6 @@ class OSNameMatches(BaseTable): ['Linux', 'Linux%']] -class ProcessTypes(BaseTable): - table = 'process_types' - columns = ['process_type'] - rows = [['browser'], - ['plugin'], - ['content']] - - class ReleaseChannels(BaseTable): table = 'release_channels' columns = ['release_channel', 'sort'] @@ -47,26 +39,6 @@ class ReleaseChannels(BaseTable): ['ESR', '5']] -class ReleaseChannelMatches(BaseTable): - table = 'release_channel_matches' - columns = ['release_channel', 'match_string'] - rows = [['Release', 'release'], - ['Release', 'default'], - ['Beta', 'beta'], - ['Aurora', 'aurora'], - ['Nightly', 'nightly%']] - - -class UptimeLevels(BaseTable): - table = 'uptime_levels' - columns = ['uptime_level', 'uptime_string', 'min_uptime', 'max_uptime'] - rows = [['1', '< 1 min', '00:00:00', '00:01:00'], - ['2', '1-5 min', '00:01:00', '00:05:00'], - ['3', '5-15 min', '00:05:00', '00:15:00'], - ['4', '15-60 min', '00:15:00', '01:00:00'], - ['5', '> 1 hour', '01:00:00', '1 year']] - - class WindowsVersions(BaseTable): table = 'windows_versions' columns = ['windows_version_name', 'major_version', 'minor_version'] @@ -138,18 +110,6 @@ class ReleaseRepositories(BaseTable): ] -class CrashTypes(BaseTable): - table = 'crash_types' - columns = [ - 'crash_type_id', 'crash_type', 'crash_type_short', 'process_type', - 'has_hang_id', 'old_code', 'include_agg'] - rows = [['1', 'Browser', 'crash', 'browser', False, 'C', True], - ['2', 'OOP Plugin', 'oop', 'plugin', False, 'P', True], - ['3', 'Hang Browser', 'hang-b', 'plugin', True, 'c', False], - ['4', 'Hang Plugin', 'hang-p', 'browser', True, 'p', True], - ['5', 'Content', 'content', 'content', False, 'T', True]] - - class ReportPartitionInfo(BaseTable): table = 'report_partition_info' columns = [ @@ -266,10 +226,10 @@ class SpecialProductPlatforms(BaseTable): ] -# the order that tables are loaded is important. -tables = [OSNames, OSNameMatches, ProcessTypes, ReleaseChannels, - ReleaseChannelMatches, UptimeLevels, WindowsVersions, +# NOTE(willkg): the order that tables are loaded is important +tables = [OSNames, OSNameMatches, ReleaseChannels, + WindowsVersions, OSVersions, ReleaseRepositories, - CrashTypes, ReportPartitionInfo, + ReportPartitionInfo, Products, ProductBuildTypes, ProductReleaseChannels, ProductProductIDMap, SpecialProductPlatforms] diff --git a/socorro/unittest/external/postgresql/test_crashstorage.py b/socorro/unittest/external/postgresql/test_crashstorage.py index 5f3e94b1b7..376fd9db55 100644 --- a/socorro/unittest/external/postgresql/test_crashstorage.py +++ b/socorro/unittest/external/postgresql/test_crashstorage.py @@ -227,11 +227,10 @@ def test_postgres_save_processed_success(self): crashstorage.save_processed(a_processed_crash) assert mocked_database_connection_source.call_count == 1 - assert mocked_cursor.execute.call_count == 2 + assert mocked_cursor.execute.call_count == 1 # check correct fragments sql_fragments = [ "UPDATE reports_20120402", - 'select id from plugins', ] for a_call, a_fragment in zip(mocked_cursor.execute.call_args_list, sql_fragments): assert a_fragment in a_call[0][0] @@ -263,12 +262,10 @@ def fetch_all_func(*args): crashstorage.save_processed(a_processed_crash) assert mocked_database_connection_source.call_count == 1 - assert mocked_cursor.execute.call_count == 3 + assert mocked_cursor.execute.call_count == 1 # check correct fragments sql_fragments = [ "UPDATE reports_20120402", - 'select id from plugins', - 'insert into plugins', ] for a_call, a_fragment in zip(mocked_cursor.execute.call_args_list, sql_fragments): assert a_fragment in a_call[0][0] @@ -294,7 +291,7 @@ def test_postgres_save_processed_success_3_truncations(self): crashstorage.save_processed(a_processed_crash_with_everything_too_long) assert mocked_database_connection_source.call_count == 1 - assert mocked_cursor.execute.call_count == 2 + assert mocked_cursor.execute.call_count == 1 # check correct fragments first_call = mocked_cursor.execute.call_args_list[0]