diff --git a/rotkehlchen/api/rest.py b/rotkehlchen/api/rest.py index a318eb6e2a..7e449d05c1 100644 --- a/rotkehlchen/api/rest.py +++ b/rotkehlchen/api/rest.py @@ -48,7 +48,7 @@ from rotkehlchen.chain.ethereum.typing import CustomEthereumToken from rotkehlchen.constants.assets import A_ETH from rotkehlchen.constants.misc import ZERO -from rotkehlchen.constants.resolver import ETHEREUM_DIRECTIVE +from rotkehlchen.constants.resolver import ethaddress_to_identifier from rotkehlchen.db.ledger_actions import DBLedgerActions from rotkehlchen.db.queried_addresses import QueriedAddresses from rotkehlchen.db.settings import ModifiableDBSettings @@ -1311,8 +1311,8 @@ def get_asset_types() -> Response: types = [str(x) for x in AssetType] return api_response(_wrap_in_ok_result(types), status_code=HTTPStatus.OK) - @staticmethod - def add_custom_asset(asset_type: AssetType, **kwargs: Any) -> Response: + @require_loggedin_user() + def add_custom_asset(self, asset_type: AssetType, **kwargs: Any) -> Response: globaldb = GlobalDBHandler() # There is no good way to figure out if an asset already exists in the DB # Best approximation we can do is this. @@ -1341,6 +1341,7 @@ def add_custom_asset(asset_type: AssetType, **kwargs: Any) -> Response: except InputError as e: return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT) + self.rotkehlchen.data.db.add_asset_identifiers([asset_id]) return api_response( _wrap_in_ok_result({'identifier': asset_id}), status_code=HTTPStatus.OK, @@ -1362,6 +1363,7 @@ def delete_custom_asset(self, identifier: str) -> Response: # Before deleting, also make sure we have up to date global DB owned data self.rotkehlchen.data.db.update_owned_assets_in_globaldb() try: + self.rotkehlchen.data.db.delete_asset_identifier(identifier) GlobalDBHandler().delete_custom_asset(identifier) except InputError as e: return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT) @@ -1391,10 +1393,9 @@ def get_custom_ethereum_tokens(address: Optional[ChecksumEthAddress]) -> Respons log_result=False, ) - @staticmethod - def add_custom_ethereum_token(token: CustomEthereumToken) -> Response: - # TODO: hacky. Clean this up when we allow addition of all assets - identifier = ETHEREUM_DIRECTIVE + token.address + @require_loggedin_user() + def add_custom_ethereum_token(self, token: CustomEthereumToken) -> Response: + identifier = ethaddress_to_identifier(token.address) try: GlobalDBHandler().add_asset( asset_id=identifier, @@ -1404,6 +1405,7 @@ def add_custom_ethereum_token(token: CustomEthereumToken) -> Response: except InputError as e: return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT) + self.rotkehlchen.data.db.add_asset_identifiers([identifier]) return api_response( _wrap_in_ok_result({'identifier': identifier}), status_code=HTTPStatus.OK, @@ -1428,7 +1430,9 @@ def edit_custom_ethereum_token(token: CustomEthereumToken) -> Response: def delete_custom_ethereum_token(self, address: ChecksumEthAddress) -> Response: # Before deleting, also make sure we have up to date global DB owned data self.rotkehlchen.data.db.update_owned_assets_in_globaldb() + try: + self.rotkehlchen.data.db.delete_asset_identifier(ethaddress_to_identifier(address)) identifier = GlobalDBHandler().delete_ethereum_token(address=address) except InputError as e: return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT) @@ -2959,6 +2963,7 @@ def _perform_assets_updates( return {'result': None, 'message': str(e), 'status_code': HTTPStatus.BAD_GATEWAY} if result is None: + self.rotkehlchen.data.db.add_globaldb_assetids() return OK_RESULT return { diff --git a/rotkehlchen/db/dbhandler.py b/rotkehlchen/db/dbhandler.py index 0e917a278d..3c38fdadf1 100644 --- a/rotkehlchen/db/dbhandler.py +++ b/rotkehlchen/db/dbhandler.py @@ -249,6 +249,7 @@ def __init__( if initial_settings is not None: self.set_settings(initial_settings) self.update_owned_assets_in_globaldb() + self.add_globaldb_assetids() def __del__(self) -> None: if hasattr(self, 'conn') and self.conn: @@ -750,8 +751,9 @@ def add_multiple_balances(self, balances: List[DBAssetBalance]) -> None: ) except sqlcipher.IntegrityError: # pylint: disable=no-member self.msg_aggregator.add_warning( - f'Tried to add a timed_balance for {entry.asset.identifier} at' - f' already existing timestamp {entry.time}. Skipping.', + f'Adding timed_balance failed. Either asset with identifier ' + f'{entry.asset.identifier} is not known or an entry for timestamp ' + f'{entry.time} already exists. Skipping.', ) continue self.conn.commit() @@ -3243,6 +3245,43 @@ def update_owned_assets_in_globaldb(self) -> None: assets = self.query_owned_assets() GlobalDBHandler().add_user_owned_assets(assets) + def add_asset_identifiers(self, asset_identifiers: List[str]) -> None: + """Adds an asset to the user db asset identifier table""" + cursor = self.conn.cursor() + cursor.executemany( + 'INSERT OR IGNORE INTO assets(identifier) VALUES(?);', + [(x,) for x in asset_identifiers], + ) + self.conn.commit() + self.update_last_write() + + def add_globaldb_assetids(self) -> None: + """Makes sure that all the GlobalDB asset identifiers are mirrored in the user DB""" + cursor = GlobalDBHandler()._conn.cursor() # after succesfull update add all asset ids + query = cursor.execute('SELECT identifier from assets;') + self.add_asset_identifiers([x[0] for x in query]) + + def delete_asset_identifier(self, asset_id: str) -> None: + """Deletes an asset identifier from the user db asset identifier table + + May raise: + - InputError if a foreign key error is encountered during deletion + """ + cursor = self.conn.cursor() + try: + cursor.execute( + 'DELETE FROM assets WHERE identifier=?;', + (asset_id,), + ) + except sqlcipher.IntegrityError as e: # pylint: disable=no-member + raise InputError( + f'Failed to delete asset with id {asset_id} from the DB since ' + f'the user owns it now or did some time in the past', + ) from e + + self.conn.commit() + self.update_last_write() + def get_latest_location_value_distribution(self) -> List[LocationData]: """Gets the latest location data diff --git a/rotkehlchen/db/schema.py b/rotkehlchen/db/schema.py index 4fae951a66..7257c9812b 100644 --- a/rotkehlchen/db/schema.py +++ b/rotkehlchen/db/schema.py @@ -142,6 +142,12 @@ INSERT OR IGNORE INTO action_type(type, seq) VALUES ('D', 4); """ +DB_CREATE_ASSETS = """ +CREATE TABLE IF NOT EXISTS assets ( + identifier TEXT NOT NULL PRIMARY KEY +); +""" + DB_CREATE_IGNORED_ACTIONS = """ CREATE TABLE IF NOT EXISTS ignored_actions ( type CHAR(1) NOT NULL DEFAULT('A') REFERENCES action_type(type), @@ -154,9 +160,10 @@ CREATE TABLE IF NOT EXISTS timed_balances ( category CHAR(1) NOT NULL DEFAULT('A') REFERENCES balance_category(category), time INTEGER, - currency VARCHAR[12], + currency TEXT, amount TEXT, usd_value TEXT, + FOREIGN KEY(currency) REFERENCES assets(identifier) ON UPDATE CASCADE, PRIMARY KEY (time, currency, category) ); """ @@ -200,13 +207,15 @@ timestamp INTEGER NOT NULL, tx_hash VARCHAR[66] NOT NULL, log_index INTEGER NOT NULL, - asset1 VARCHAR[12] NOT NULL, + asset1 TEXT NOT NULL, asset1_amount TEXT NOT NULL, asset1_usd_value TEXT NOT NULL, - asset2 VARCHAR[12], + asset2 TEXT, asset2amount_borrowrate_feeamount TEXT, asset2usd_value_accruedinterest_feeusdvalue TEXT, borrow_rate_mode VARCHAR[10], + FOREIGN KEY(asset1) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(asset2) REFERENCES assets(identifier) ON UPDATE CASCADE, PRIMARY KEY (event_type, tx_hash, log_index) ); """ @@ -215,10 +224,10 @@ CREATE TABLE IF NOT EXISTS yearn_vaults_events ( address VARCHAR[42] NOT NULL, event_type VARCHAR[10] NOT NULL, - from_asset VARCHAR[44] NOT NULL, + from_asset TEXT NOT NULL, from_amount TEXT NOT NULL, from_usd_value TEXT NOT NULL, - to_asset VARCHAR[44] NOT NULL, + to_asset TEXT NOT NULL, to_amount TEXT NOT NULL, to_usd_value TEXT NOT NULL, pnl_amount TEXT, @@ -227,6 +236,8 @@ timestamp INTEGER NOT NULL, tx_hash VARCHAR[66] NOT NULL, log_index INTEGER NOT NULL, + FOREIGN KEY(from_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(to_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, PRIMARY KEY (event_type, tx_hash, log_index) ); """ @@ -287,10 +298,11 @@ DB_CREATE_MANUALLY_TRACKED_BALANCES = """ CREATE TABLE IF NOT EXISTS manually_tracked_balances ( - asset VARCHAR[24] NOT NULL, + asset TEXT NOT NULL, label TEXT NOT NULL PRIMARY KEY, amount TEXT, - location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location) + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE ); """ @@ -324,9 +336,12 @@ amount TEXT NOT NULL, rate TEXT NOT NULL, fee TEXT, - fee_currency TEXT[10], + fee_currency TEXT, link TEXT, - notes TEXT + notes TEXT, + FOREIGN KEY(base_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(quote_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE ); """ @@ -337,11 +352,13 @@ open_time INTEGER, close_time INTEGER, profit_loss TEXT, - pl_currency VARCHAR[10], + pl_currency TEXT NOT NULL, fee TEXT, - fee_currency VARCHAR[10], + fee_currency TEXT, link TEXT, - notes TEXT + notes TEXT, + FOREIGN KEY(pl_currency) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE ); """ @@ -353,11 +370,13 @@ address TEXT, transaction_id TEXT, time INTEGER, - asset VARCHAR[10], + asset TEXT NOT NULL, amount TEXT, - fee_asset VARCHAR[10], + fee_asset TEXT, fee TEXT, - link TEXT + link TEXT, + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_asset) REFERENCES assets(identifier) ON UPDATE CASCADE ); """ @@ -372,7 +391,9 @@ rate TEXT, rate_asset TEXT, link TEXT, - notes TEXT + notes TEXT, + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(rate_asset) REFERENCES assets(identifier) ON UPDATE CASCADE ); """ @@ -517,6 +538,7 @@ channel_id TEXT, token TEXT, log_index INTEGER, + FOREIGN KEY(token) REFERENCES assets(identifier) ON UPDATE CASCADE, PRIMARY KEY (tx_hash, address, type, log_index) ); """ @@ -603,7 +625,7 @@ DB_SCRIPT_CREATE_TABLES = """ PRAGMA foreign_keys=off; BEGIN TRANSACTION; -{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{} +{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{} COMMIT; PRAGMA foreign_keys=on; """.format( @@ -611,6 +633,7 @@ DB_CREATE_LOCATION, DB_CREATE_ASSET_MOVEMENT_CATEGORY, DB_CREATE_BALANCE_CATEGORY, + DB_CREATE_ASSETS, DB_CREATE_TIMED_BALANCES, DB_CREATE_TIMED_LOCATION_DATA, DB_CREATE_USER_CREDENTIALS, diff --git a/rotkehlchen/db/upgrade_manager.py b/rotkehlchen/db/upgrade_manager.py index b7e982c40b..56517ccf93 100644 --- a/rotkehlchen/db/upgrade_manager.py +++ b/rotkehlchen/db/upgrade_manager.py @@ -262,7 +262,7 @@ def _perform_single_upgrade(self, upgrade: UpgradeRecord) -> None: raise DBUpgradeError(error_message) from e # for some upgrades even for success keep the backup of the previous db - if upgrade.from_version == 24: + if upgrade.from_version in (24, 25): shutil.copyfile( tmp_db_path, os.path.join(self.db.user_data_dir, tmp_db_filename), diff --git a/rotkehlchen/db/upgrades/v25_v26.py b/rotkehlchen/db/upgrades/v25_v26.py index 00f425940a..2c6714fffa 100644 --- a/rotkehlchen/db/upgrades/v25_v26.py +++ b/rotkehlchen/db/upgrades/v25_v26.py @@ -1,5 +1,6 @@ -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Set +from rotkehlchen.globaldb.handler import GlobalDBHandler from rotkehlchen.user_messages import MessagesAggregator if TYPE_CHECKING: @@ -30,6 +31,11 @@ def __init__(self, msg_aggregator: MessagesAggregator) -> None: 'ftx': 'Z', 'rotkehlchen': 'A', } + globaldb = GlobalDBHandler() + globaldb_conn = globaldb._conn + globaldb_cursor = globaldb_conn.cursor() + query = globaldb_cursor.execute('SELECT identifier from assets;') + self.all_asset_ids = {x[0] for x in query} def upgrade_user_credentials(self, cursor: 'Cursor') -> None: # get old data @@ -108,6 +114,378 @@ def purge_binanceus(cursor: 'Cursor') -> None: cursor.execute('DELETE FROM trades WHERE location = "S";') cursor.execute('DELETE FROM asset_movements WHERE location = "S";') + def _process_asset_identifiers( + self, + asset_ids: Set[str], + table_name: str, + ) -> bool: + """Processes the given asset ids of an entry and determines if the entry can be used. + + If an asset can't be found in the global db False is returned and warning logged. + """ + asset_ids = asset_ids - {None} # remove empty identifiers + if asset_ids.issubset(self.all_asset_ids): + return True + + for asset_id in asset_ids: + if asset_id not in self.all_asset_ids: + self.msg_aggregator.add_warning( + f'During v25 -> v26 DB upgrade found {table_name} entry of unknown asset ' + f'{asset_id}. This will not be in the upgraded DB.', + ) + return False + + return True + + def upgrade_timed_balances(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT category, time, currency, amount, usd_value FROM timed_balances;', + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS timed_balances;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[2]}, + table_name='timed_balances', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS timed_balances ( + category CHAR(1) NOT NULL DEFAULT('A') REFERENCES balance_category(category), + time INTEGER, + currency TEXT, + amount TEXT, + usd_value TEXT, + FOREIGN KEY(currency) REFERENCES assets(identifier) ON UPDATE CASCADE, + PRIMARY KEY (time, currency, category) + ); + """) + cursor.executemany( + 'INSERT INTO timed_balances(category, time, currency, amount, usd_value) ' + 'VALUES(?, ?, ?, ?, ?);', + new_tuples, + ) + + def upgrade_manually_tracked_balances(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT asset, label, amount, location FROM manually_tracked_balances;', + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS manually_tracked_balances;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[0]}, + table_name='manually_tracked_balances', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS manually_tracked_balances ( + asset TEXT NOT NULL, + label TEXT NOT NULL PRIMARY KEY, + amount TEXT, + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE + ); + """) + cursor.executemany( + 'INSERT INTO manually_tracked_balances(asset, label, amount, location) ' + 'VALUES(?, ?, ?, ?);', + new_tuples, + ) + + def upgrade_margin_positions(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT id, location, open_time, close_time, profit_loss, pl_currency, fee, fee_currency, link, notes FROM margin_positions;', # noqa: E501 + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS margin_positions;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[5], entry[7]}, + table_name='margin_positions', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS margin_positions ( + id TEXT PRIMARY KEY, + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + open_time INTEGER, + close_time INTEGER, + profit_loss TEXT, + pl_currency TEXT NOT NULL, + fee TEXT, + fee_currency TEXT, + link TEXT, + notes TEXT, + FOREIGN KEY(pl_currency) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE + ); + """) + cursor.executemany( + 'INSERT INTO margin_positions(' + 'id, location, open_time, close_time, profit_loss, pl_currency, fee, ' + 'fee_currency, link, notes' + ') VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', + new_tuples, + ) + + def upgrade_asset_movements(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT id, location, category, address, transaction_id, time, asset, amount, fee_asset, fee, link FROM asset_movements;', # noqa: E501 + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS asset_movements;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[6], entry[8]}, + table_name='asset_movements', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS asset_movements ( + id TEXT PRIMARY KEY, + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + category CHAR(1) NOT NULL DEFAULT('A') REFERENCES asset_movement_category(category), + address TEXT, + transaction_id TEXT, + time INTEGER, + asset TEXT NOT NULL, + amount TEXT, + fee_asset TEXT, + fee TEXT, + link TEXT, + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_asset) REFERENCES assets(identifier) ON UPDATE CASCADE + ); + """) + cursor.executemany( + 'INSERT INTO asset_movements(' + 'id, location, category, address, transaction_id, time, ' + 'asset, amount, fee_asset, fee, link' + ') VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', + new_tuples, + ) + + def upgrade_ledger_actions(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT identifier, timestamp, type, location, amount, asset, rate, rate_asset, link, notes FROM ledger_actions;', # noqa: E501 + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS ledger_actions;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[5], entry[7]}, + table_name='ledger_actions', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS ledger_actions ( + identifier INTEGER NOT NULL PRIMARY KEY, + timestamp INTEGER NOT NULL, + type CHAR(1) NOT NULL DEFAULT('A') REFERENCES ledger_action_type(type), + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + amount TEXT NOT NULL, + asset TEXT NOT NULL, + rate TEXT, + rate_asset TEXT, + link TEXT, + notes TEXT, + FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(rate_asset) REFERENCES assets(identifier) ON UPDATE CASCADE + ); + """) + cursor.executemany( + 'INSERT INTO ledger_actions(' + 'identifier, timestamp, type, location, amount, asset, rate, rate_asset, link, notes' + ') VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', + new_tuples, + ) + + def upgrade_trades(self, cursor: 'Cursor') -> None: + query = cursor.execute( + 'SELECT id, time, location, base_asset, quote_asset, type, amount, rate, fee, fee_currency, link, notes FROM trades;', # noqa: E501 + ) + old_tuples = query.fetchall() + cursor.execute('DROP TABLE IF EXISTS trades;') + new_tuples = [] + for entry in old_tuples: + should_write = self._process_asset_identifiers( + asset_ids={entry[3], entry[4], entry[9]}, + table_name='trades', + ) + if not should_write: + continue + + new_tuples.append(entry) + + # create the new table and insert all values into it + cursor.execute(""" + CREATE TABLE IF NOT EXISTS trades ( + id TEXT PRIMARY KEY NOT NULL, + time INTEGER NOT NULL, + location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location), + base_asset TEXT NOT NULL, + quote_asset TEXT NOT NULL, + type CHAR(1) NOT NULL DEFAULT ('A') REFERENCES trade_type(type), + amount TEXT NOT NULL, + rate TEXT NOT NULL, + fee TEXT, + fee_currency TEXT, + link TEXT, + notes TEXT, + FOREIGN KEY(base_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(quote_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE + ); + """) + cursor.executemany( + 'INSERT INTO trades(' + 'id, time, location, base_asset, quote_asset, type, ' + 'amount, rate, fee, fee_currency, link, notes' + ') VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);', + new_tuples, + ) + + def introduce_assets_table(self, cursor: 'Cursor') -> None: + """ + Does the migration to the assets table. + https://github.com/rotki/rotki/issues/2906 + This is a table containing all assets owned by the user and creates foreign + key relations from all other tables to it. + + Tables containing asset identifiers. [X] -> should not be deleted and repulled + + - adex_events + - aave_events + - yearn_vaults_events + - ethereum_accounts_details + - timed_balances [X] + - manually_tracked_balances [X] + - margin_positions [X] + - asset_movements [X] + - ledger_actions [X] + - trades [X] + + Tables that are not touched due to the unknown token construct: + - amm_swaps + - uniswap_events + - balancer pools + + -> Remember to also clear relevant used_query_ranges for the deleted tables + """ + # First populate all assets table with all identifiers of the global db assets + cursor.executemany( + 'INSERT OR IGNORE INTO assets(identifier) VALUES(?);', + [(x,) for x in self.all_asset_ids], + ) + # Then just drop and recreate tables that can easily be repulled + cursor.execute('DROP TABLE IF EXISTS adex_events;') + cursor.execute('DELETE FROM used_query_ranges WHERE name LIKE "adex_events%";') + cursor.execute(""" + CREATE TABLE IF NOT EXISTS adex_events ( + tx_hash VARCHAR[42] NOT NULL, + address VARCHAR[42] NOT NULL, + identity_address VARCHAR[42] NOT NULL, + timestamp INTEGER NOT NULL, + type TEXT NOT NULL, + pool_id TEXT NOT NULL, + amount TEXT NOT NULL, + usd_value TEXT NOT NULL, + bond_id TEXT, + nonce INT, + slashed_at INTEGER, + unlock_at INTEGER, + channel_id TEXT, + token TEXT, + log_index INTEGER, + FOREIGN KEY(token) REFERENCES assets(identifier) ON UPDATE CASCADE, + PRIMARY KEY (tx_hash, address, type, log_index) + ); + """) + cursor.execute('DROP TABLE IF EXISTS aave_events;') + cursor.execute('DELETE FROM used_query_ranges WHERE name LIKE "aave_events%";') + cursor.execute(""" + CREATE TABLE IF NOT EXISTS aave_events ( + address VARCHAR[42] NOT NULL, + event_type VARCHAR[10] NOT NULL, + block_number INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + tx_hash VARCHAR[66] NOT NULL, + log_index INTEGER NOT NULL, + asset1 TEXT NOT NULL, + asset1_amount TEXT NOT NULL, + asset1_usd_value TEXT NOT NULL, + asset2 TEXT, + asset2amount_borrowrate_feeamount TEXT, + asset2usd_value_accruedinterest_feeusdvalue TEXT, + borrow_rate_mode VARCHAR[10], + FOREIGN KEY(asset1) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(asset2) REFERENCES assets(identifier) ON UPDATE CASCADE, + PRIMARY KEY (event_type, tx_hash, log_index) + ); + """) + cursor.execute('DROP TABLE IF EXISTS yearn_vaults_events;') + cursor.execute('DELETE FROM used_query_ranges WHERE name LIKE "yearn_vaults_events%";') + cursor.execute(""" + CREATE TABLE IF NOT EXISTS yearn_vaults_events ( + address VARCHAR[42] NOT NULL, + event_type VARCHAR[10] NOT NULL, + from_asset TEXT NOT NULL, + from_amount TEXT NOT NULL, + from_usd_value TEXT NOT NULL, + to_asset TEXT NOT NULL, + to_amount TEXT NOT NULL, + to_usd_value TEXT NOT NULL, + pnl_amount TEXT, + pnl_usd_value TEXT, + block_number INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + tx_hash VARCHAR[66] NOT NULL, + log_index INTEGER NOT NULL, + FOREIGN KEY(from_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + FOREIGN KEY(to_asset) REFERENCES assets(identifier) ON UPDATE CASCADE, + PRIMARY KEY (event_type, tx_hash, log_index) + ); + """) + cursor.execute('DELETE FROM ethereum_accounts_details;') + + self.upgrade_timed_balances(cursor) + self.upgrade_manually_tracked_balances(cursor) + self.upgrade_margin_positions(cursor) + self.upgrade_asset_movements(cursor) + self.upgrade_ledger_actions(cursor) + self.upgrade_trades(cursor) + def upgrade_v25_to_v26(db: 'DBHandler') -> None: """Upgrades the DB from v25 to v26 @@ -121,6 +499,7 @@ def upgrade_v25_to_v26(db: 'DBHandler') -> None: For that reason we need to purge used query ranges and data of binance_us - Delete the unused anonymized logs setting from the DB + - Introduce assets table and foreign key relationships for assets """ helper = V25V26UpgradeHelper(db.msg_aggregator) cursor = db.conn.cursor() @@ -128,5 +507,6 @@ def upgrade_v25_to_v26(db: 'DBHandler') -> None: helper.migrate_kraken_account_type(cursor) helper.purge_binanceus(cursor) cursor.execute('DELETE from settings WHERE name="anonymized_logs";') + helper.introduce_assets_table(cursor) del helper db.conn.commit() diff --git a/rotkehlchen/tests/api/test_custom_assets.py b/rotkehlchen/tests/api/test_custom_assets.py index b003bb5fb6..79b4b8d924 100644 --- a/rotkehlchen/tests/api/test_custom_assets.py +++ b/rotkehlchen/tests/api/test_custom_assets.py @@ -16,7 +16,7 @@ @pytest.mark.parametrize('use_clean_caching_directory', [True]) -@pytest.mark.parametrize('start_with_logged_in_user', [False]) +@pytest.mark.parametrize('start_with_logged_in_user', [True]) def test_adding_custom_assets(rotkehlchen_api_server, globaldb): """Test that the endpoint for adding a custom asset works""" @@ -136,7 +136,7 @@ def test_adding_custom_assets(rotkehlchen_api_server, globaldb): @pytest.mark.parametrize('use_clean_caching_directory', [True]) -@pytest.mark.parametrize('start_with_logged_in_user', [False]) +@pytest.mark.parametrize('start_with_logged_in_user', [True]) def test_editing_custom_assets(rotkehlchen_api_server, globaldb): """Test that the endpoint for editing a custom asset works""" @@ -417,10 +417,7 @@ def test_custom_asset_delete_guard(rotkehlchen_api_server): ), json={'identifier': custom1_id}, ) - expected_msg = ( - 'Tried to delete asset with name "foo token" and symbol "FOO" but its deletion ' - 'would violate a constraint' - ) + expected_msg = 'Failed to delete asset with id' assert_error_response( response=response, contained_in_msg=expected_msg, diff --git a/rotkehlchen/tests/api/test_custom_ethereum_tokens.py b/rotkehlchen/tests/api/test_custom_ethereum_tokens.py index 5c51af0b29..89feecdef8 100644 --- a/rotkehlchen/tests/api/test_custom_ethereum_tokens.py +++ b/rotkehlchen/tests/api/test_custom_ethereum_tokens.py @@ -92,7 +92,7 @@ def test_query_custom_tokens(rotkehlchen_api_server): @pytest.mark.parametrize('use_clean_caching_directory', [True]) -@pytest.mark.parametrize('start_with_logged_in_user', [False]) +@pytest.mark.parametrize('start_with_logged_in_user', [True]) @pytest.mark.parametrize('custom_ethereum_tokens', [INITIAL_TOKENS]) def test_adding_custom_tokens(rotkehlchen_api_server): """Test that the endpoint for adding a custom ethereum token works""" @@ -233,7 +233,7 @@ def test_adding_custom_tokens(rotkehlchen_api_server): @pytest.mark.parametrize('use_clean_caching_directory', [True]) -@pytest.mark.parametrize('start_with_logged_in_user', [False]) +@pytest.mark.parametrize('start_with_logged_in_user', [True]) @pytest.mark.parametrize('custom_ethereum_tokens', [INITIAL_TOKENS]) def test_editing_custom_tokens(rotkehlchen_api_server): """Test that the endpoint for editing a custom ethereum token works""" @@ -348,7 +348,7 @@ def test_deleting_custom_tokens(rotkehlchen_api_server): ) expected_msg = ( f'Tried to delete ethereum token with address {non_existing_address} ' - f'but it was not found in the DB', + f'but it was not found in the DB' ) assert_error_response( response=response, @@ -462,10 +462,7 @@ def test_custom_tokens_delete_guard(rotkehlchen_api_server): ), json={'address': INITIAL_TOKENS[0].address}, ) - expected_msg = ( - f'Tried to delete ethereum token with address {INITIAL_TOKENS[0].address} ' - f'but its deletion would violate a constraint so deletion failed.' - ) + expected_msg = 'Failed to delete asset with id' assert_error_response( response=response, contained_in_msg=expected_msg, diff --git a/rotkehlchen/tests/data/v25_rotkehlchen.db b/rotkehlchen/tests/data/v25_rotkehlchen.db index 6ad4a1487d..9e55133b2f 100644 Binary files a/rotkehlchen/tests/data/v25_rotkehlchen.db and b/rotkehlchen/tests/data/v25_rotkehlchen.db differ diff --git a/rotkehlchen/tests/db/test_db.py b/rotkehlchen/tests/db/test_db.py index a768c0132f..6940bde76f 100644 --- a/rotkehlchen/tests/db/test_db.py +++ b/rotkehlchen/tests/db/test_db.py @@ -86,6 +86,7 @@ from rotkehlchen.utils.serialization import rlk_jsondumps TABLES_AT_INIT = [ + 'assets', 'aave_events', 'yearn_vaults_events', 'timed_balances', @@ -619,13 +620,6 @@ def test_query_owned_assets(data_dir, username): ), ]) data.db.add_multiple_balances(balances) - cursor = data.db.conn.cursor() - cursor.execute( - 'INSERT INTO timed_balances(' - ' time, currency, amount, usd_value, category) ' - ' VALUES(?, ?, ?, ?, ?)', - (1469326500, 'ADSADX', '10.1', '100.5', 'A'), - ) data.db.conn.commit() # also make sure that assets from trades are included @@ -697,8 +691,7 @@ def test_query_owned_assets(data_dir, username): assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH} # noqa: E501 assert all(isinstance(x, Asset) for x in assets_list) warnings = data.db.msg_aggregator.consume_warnings() - assert len(warnings) == 1 - assert 'Unknown/unsupported asset ADSADX' in warnings[0] + assert len(warnings) == 0 def test_get_latest_location_value_distribution(data_dir, username): diff --git a/rotkehlchen/tests/db/test_db_upgrades.py b/rotkehlchen/tests/db/test_db_upgrades.py index 6f405464e0..1efc0be6e3 100644 --- a/rotkehlchen/tests/db/test_db_upgrades.py +++ b/rotkehlchen/tests/db/test_db_upgrades.py @@ -10,6 +10,8 @@ from rotkehlchen.accounting.structures import BalanceType from rotkehlchen.assets.asset import Asset +from rotkehlchen.assets.typing import AssetType +from rotkehlchen.chain.ethereum.typing import CustomEthereumToken from rotkehlchen.data_handler import DataHandler from rotkehlchen.db.dbhandler import DBHandler from rotkehlchen.db.old_create import OLD_DB_SCRIPT_CREATE_TABLES @@ -27,8 +29,12 @@ ) from rotkehlchen.db.upgrades.v13_v14 import REMOVED_ASSETS, REMOVED_ETH_TOKENS from rotkehlchen.errors import DBUpgradeError -from rotkehlchen.tests.utils.database import mock_dbhandler_update_owned_assets +from rotkehlchen.tests.utils.database import ( + mock_dbhandler_add_globaldb_assetids, + mock_dbhandler_update_owned_assets, +) from rotkehlchen.tests.utils.factories import make_ethereum_address +from rotkehlchen.typing import ChecksumEthAddress from rotkehlchen.user_messages import MessagesAggregator creation_patch = patch( @@ -316,7 +322,7 @@ def test_upgrade_db_1_to_2(data_dir, username): ethereum accounts are now checksummed""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) - with creation_patch, target_patch(1), mock_dbhandler_update_owned_assets(): + with creation_patch, target_patch(1), mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids(): # noqa: E501 data.unlock(username, '123', create_new=True) # Manually input a non checksummed account data.db.conn.commit() @@ -345,14 +351,14 @@ def test_upgrade_db_1_to_2(data_dir, username): def test_upgrade_db_2_to_3(user_data_dir): """Test upgrading the DB from version 2 to version 3, rename BCHSV to BSV""" msg_aggregator = MessagesAggregator() - with creation_patch: + with creation_patch, mock_dbhandler_add_globaldb_assetids(): db = _init_db_with_target_version( target_version=2, user_data_dir=user_data_dir, msg_aggregator=msg_aggregator, ) - with mock_dbhandler_update_owned_assets(): + with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids(): populate_db_and_check_for_asset_renaming( db=db, user_data_dir=user_data_dir, @@ -369,7 +375,7 @@ def test_upgrade_db_3_to_4(data_dir, username): the eth_rpc_port setting is changed to eth_rpc_endpoint""" msg_aggregator = MessagesAggregator() data = DataHandler(data_dir, msg_aggregator) - with creation_patch, target_patch(3), mock_dbhandler_update_owned_assets(): + with creation_patch, target_patch(3), mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids(): # noqa: E501 data.unlock(username, '123', create_new=True) # Manually set version and input the old rpcport setting cursor = data.db.conn.cursor() @@ -384,7 +390,7 @@ def test_upgrade_db_3_to_4(data_dir, username): data.db.conn.commit() # now relogin and check that the setting has been changed and the version bumped - with mock_dbhandler_update_owned_assets(): + with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids(): del data data = DataHandler(data_dir, msg_aggregator) with target_patch(target_version=4): @@ -406,14 +412,14 @@ def test_upgrade_db_3_to_4(data_dir, username): def test_upgrade_db_4_to_5(user_data_dir): """Test upgrading the DB from version 4 to version 5, rename BCC to BCH""" msg_aggregator = MessagesAggregator() - with creation_patch: + with creation_patch, mock_dbhandler_add_globaldb_assetids(): db = _init_db_with_target_version( target_version=4, user_data_dir=user_data_dir, msg_aggregator=msg_aggregator, ) - with mock_dbhandler_update_owned_assets(): + with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids(): populate_db_and_check_for_asset_renaming( db=db, user_data_dir=user_data_dir, @@ -1772,12 +1778,24 @@ def test_upgrade_db_24_to_25(user_data_dir): # pylint: disable=unused-argument @pytest.mark.parametrize('use_clean_caching_directory', [True]) @pytest.mark.parametrize('have_kraken', [True, False]) @pytest.mark.parametrize('have_kraken_setting', [True, False]) -def test_upgrade_db_25_to_26(user_data_dir, have_kraken, have_kraken_setting): # pylint: disable=unused-argument # noqa: E501 +def test_upgrade_db_25_to_26(globaldb, user_data_dir, have_kraken, have_kraken_setting): # pylint: disable=unused-argument # noqa: E501 """Test upgrading the DB from version 25 to version 26""" msg_aggregator = MessagesAggregator() v25_conn = _init_prepared_db(user_data_dir, 'v25_rotkehlchen.db') cursor = v25_conn.cursor() + # make sure the globaldb has a custom token used in the DB + globaldb.add_asset( + asset_id='_ceth_0x48Fb253446873234F2fEBbF9BdeAA72d9d387f94', + asset_type=AssetType.ETHEREUM_TOKEN, + data=CustomEthereumToken( + address=ChecksumEthAddress('0x48Fb253446873234F2fEBbF9BdeAA72d9d387f94'), + decimals=18, + name='foo', + symbol='FOO', + ), + ) + # Checks before migration credentials = cursor.execute( 'SELECT name, api_key, api_secret, passphrase from user_credentials', @@ -1822,7 +1840,8 @@ def test_upgrade_db_25_to_26(user_data_dir, have_kraken, have_kraken_setting): ).fetchone() assert settings == ('anonymized_logs', 'True') - # check trades/assets movements and used query ranges are there + # check all tables are there before the upgrade + assert cursor.execute('SELECT COUNT(*) from timed_balances;').fetchone()[0] == 392 trades_query = cursor.execute( 'SELECT id, time, location, base_asset, quote_asset, type, amount, rate, fee,' 'fee_currency, link, notes from trades ORDER BY time ASC;', @@ -1830,12 +1849,66 @@ def test_upgrade_db_25_to_26(user_data_dir, have_kraken, have_kraken_setting): assert trades_query == [ ('foo1', 1, 'S', 'ETH', 'BTC', 'A', '1', '1', '1', 'ETH', '', ''), ('foo2', 1, 'B', 'ETH', 'BTC', 'A', '1', '1', '1', 'ETH', '', ''), + ('fac279d109466a908119816d2ee7af90fba28f1ae60437bcbfab10e40a62bbc7', 1493227049, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '867.46673624', '0.00114990', '0.00249374', 'BTC', 'l1', None), # noqa: E501 + ('56775b4b80f46d1dfc1b53fc0f6b61573c14142499bfe3a62e3371f7afc166db', 1493228252, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'A', '11348.12286689', '0.00008790', '0.00249374', 'BTC', 'l2', None), # noqa: E501 + ('d38687c92fd4b56f7241b38653390a72022709ef8835c289f6d49cc436b8e05a', 1498949799, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '150.00000000', '0.00091705', '0.00034389', 'BTC', 'l3', None), # noqa: E501 + ('cc31283b6e723f4a7364496b349869564b06b4320da3a14d95dcda1801369361', 1498966605, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '1000.00000000', '0.00091409', '0.00228521', 'BTC', 'l4', None), # noqa: E501 + ('c76a42bc8b32407b7444da89cc9f73c22fd6a1bc0055b7d4112e3e59709b2b5b', 1499011364, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '200.00000000', '0.00092401', '0.00046200', 'BTC', 'l5', None), # noqa: E501 + ('18810df423b24bb438360b23b50bd0fc11ed2d3701420651ef716f4840367894', 1499051024, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '120.00000000', '0.00091603', '0.00027480', 'BTC', 'l6', None), # noqa: E501 + ('27bb15dfc1a008c2efa2c5510b30808d8246ab903cf9f950ea7a175f0779925d', 1499675187, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '200.00000000', '0.00091747', '0.00045869', 'BTC', 'l7', None), # noqa: E501 + ('b0c4d1d816fa3448ba1ab1a285a35d71a42ff27d68438626eacecc4bf927ab07', 1499677638, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '1135.00000000', '0.00088101', '0.00249985', 'BTC', 'l8', None), # noqa: E501 + ('6b91fc81d40c121af7397c5f3351547de4bd3f288fe483ace14b7b7922cfcd36', 1500494329, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'B', '10490.34064784', '0.00024385', '0.00639521', 'BTC', 'l9', None), # noqa: E501 + ('59ea4f98a815467122d201d737b97f910c8918cfe8f476a74d51a4006ef1aaa8', 1500501003, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'B', '857.78221905', '0.00018099', '0.00038812', 'BTC', 'l10', None), # noqa: E501 + ('dc4a8a1dd3ef78b6eae5ee69f24bed73196ba3677150015f80d28a70b963253c', 1501194075, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '510.21713000', '0.00039101', '0.00049875', 'BTC', 'l11', None), # noqa: E501 + ('8ba98869f1398d9d64d974a13bc1e686746c7068b765223a46f886ef9c100722', 1501585385, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '731.28354007', '0.00034101', '0.00062343', 'BTC', 'l12', None), # noqa: E501 + ('6f583c1297a86beadb46e8876db9589bb2ca60c603a4eca77b331611a49cd482', 1599751485, 'A', 'ETH', 'EUR', 'A', '150', '210.15', '0.2', 'EUR', None, None), # noqa: E501 + ('3fc94b6b91de61d98b21bdba9a6e449b3ff25756f34a14d17dcf3979d08c4ee3', 1607172606, 'D', 'MAID', 'BTC', 'B', '15515.00000000', '0.00001299', '0.00040305', 'BTC', 'l13', None), # noqa: E501 + ('ecd64dba4367a42292988abb34ed46b3dda0d48728c629f9727706d198023d6c', 1610915040, 'A', 'ETH', 'EUR', 'A', '5', '0.1', '0.001', '_ceth_0x111111111117dC0aa78b770fA6A738034120C302', 'dsad', 'ads'), # noqa: E501c + ('7aae102d9240f7d5f7f0669d6eefb47f2d1cf5bba462b0cee267719e9272ffde', 1612302374, 'A', '_ceth_0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984', 'ETH', 'A', '1', '0.01241', '0', '_ceth_0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984', None, None), # noqa: E501 ] asset_movements_query = cursor.execute('SELECT id, location, category, address, transaction_id, time, asset, amount, fee_asset, fee, link from asset_movements ORDER BY time ASC;').fetchall() # noqa: E501 assert asset_movements_query == [ - ('foo1', 'S', 'B', '0xaddy', '0xtxid', 1, 'YFI', '1', 'GNO', '1', 'customlink'), - ('foo2', 'B', 'B', '0xaddy', '0xtxid', 1, 'YFI', '1', 'GNO', '1', 'customlink'), + ('foo1', 'S', 'B', '0xaddy', '0xtxid', 1, 'ETH', '1', 'BTC', '1', 'customlink'), + ('foo2', 'B', 'B', '0xaddy', '0xtxid', 1, 'ETH', '1', '_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F', '1', 'customlink'), # noqa: E501 + ('822511b6035c5d2a7a7ff82c21b61381016e76764e84f656aedcfbc3b7a2e2f4', 'L', 'B', '0xaddy', '0xtxid', 1, '_ceth_0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e', '1', '_ceth_0x6810e776880C02933D47DB1b9fc05908e5386b96', '1', 'customlink'), # noqa: E501 + ('98c8378892955d3c95cc24277188ad33504d2e668441ba23a270b6c65f00be43', 'D', 'A', '0xaddress', '0xtxid', 1493226738, 'BTC', '2.00000000', 'BTC', '0', 'link3'), # noqa: E501 + ('9713d2c2f90edfc375bfea1d014599e9f3a20eded94625c0a2483c4ab2692ff9', 'D', 'A', '0xaddress', '0xtxid', 1498941726, 'BTC', '4.20000000', 'BTC', '0', 'link2'), # noqa: E501 + ('86f6cda4bcd36e2fd0e8938fd3b31ebe895af2df6d8b60479c401cd846a3ccf8', 'D', 'B', '0xaddress', '0xtxid', 1501161076, 'BTC', '3.91944853', 'BTC', '0.00100000', 'link5'), # noqa: E501 + ('9a3ab62aea2892e9000c868ce29a471e34f57d3bbae7691b920bcf58fbea10ce', 'D', 'A', '0xaddress', '0xtxid', 1577666912, 'MAID', '15515.00000000', 'MAID', '0', 'link1'), # noqa: E501 + ('79d6d91d1fd2acf02a9d244e33ff340c04a938faaf0d1ba10aba9d8ae55b11cc', 'D', 'B', '0xaddress', '0xtxid', 1607094370, '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', '4753.96740631', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', '160.00000000', 'link4'), # noqa: E501 + ] + # Check manually tracked balances before upgrade + query = cursor.execute('SELECT asset, label, amount, location from manually_tracked_balances;') # noqa: E501 + assert query.fetchall() == [ + ('EUR', 'test eur balance', '1', 'A'), + ('USD', 'test usd balance', '1', 'A'), + ('CNY', 'test CNY balance', '1', 'A'), + ('_ceth_0x8Ab7404063Ec4DBcfd4598215992DC3F8EC853d7', 'exotic asset', '1500', 'A'), + ('_ceth_0x111111111117dC0aa78b770fA6A738034120C302', 'test for duplication', '100000', 'J'), # noqa: E501 + ('_ceth_0x48Fb253446873234F2fEBbF9BdeAA72d9d387f94', 'test custom token balance', '65', 'A'), # noqa: E501 + ('_ceth_0x50D1c9771902476076eCFc8B2A83Ad6b9355a4c9', 'test_asset_with_same_symbol', '85', 'A'), # noqa: E501 + ('_ceth_0xdb89d55d8878680FED2233ea6E1Ae7DF79C7073e', 'test_custom_token', '25', 'A'), # since this token is not known this won't be here after the upgrade # noqa: E501 + ] + # Check ledger actions before upgrade + query = cursor.execute('SELECT identifier, timestamp, type, location, amount, asset, rate, rate_asset, link, notes from ledger_actions;') # noqa: E501 + assert query.fetchall() == [ + (1, 1611260690, 'A', 'A', '1', '_ceth_0x0E8d6b471e332F140e7d9dbB99E5E3822F728DA6', None, None, None, None), # noqa: E501 + (2, 1610483475, 'A', 'A', '1', '_ceth_0xB6eD7644C69416d67B522e20bC294A9a9B405B31', None, None, 'sad', 'asdsad'), # noqa: E501 ] + # Check margin positions before upgrade + query = cursor.execute('SELECT id, location, open_time, close_time, profit_loss, pl_currency, fee, fee_currency, link, notes from margin_positions;') # noqa: E501 + raw_upgraded = query.fetchall() + assert raw_upgraded == [ + ("3ebd1ff33f6b6431778db56393e6105b94b0b23d0976462d70279e6f82db9924", "C", 1, 5, "500", "ETH", "1", "_ceth_0x6810e776880C02933D47DB1b9fc05908e5386b96", "", ""), # noqa: E501 + ("2ecdf50622f0ad6277b2c4b28954118753db195c9ae2005ce7da7b30d4a873c4", "A", 1, 5, "1", "BTC", "1", "_ceth_0x255Aa6DF07540Cb5d3d297f0D0D4D84cb52bc8e6", "", ""), # noqa: E501 + ("bec34827cd9ce879e91d45dfe11942752f810504439701ff7f3d005850f458a8", "A", 0, 5, "1", "_ceth_0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e", "1", "_ceth_0x1494CA1F11D487c2bBe4543E90080AeBa4BA3C2b", "", ""), # noqa: E501 + ] + # Check the tables that are just gonna get deleted have data before the upgrade + assert cursor.execute('SELECT COUNT(*) from adex_events;').fetchone()[0] == 1 + assert cursor.execute('SELECT COUNT(*) from aave_events;').fetchone()[0] == 1 + assert cursor.execute('SELECT COUNT(*) from yearn_vaults_events;').fetchone()[0] == 1 + assert cursor.execute('SELECT COUNT(*) from ethereum_accounts_details;').fetchone()[0] == 1 + # Check used query ranges before upgrade used_query_ranges = cursor.execute('SELECT * from used_query_ranges').fetchall() assert used_query_ranges == [ ('binance_us_trades', 0, 1), @@ -1898,18 +1971,123 @@ def test_upgrade_db_25_to_26(user_data_dir, have_kraken, have_kraken_setting): assert mapping is None # check trades/assets movements and used query ranges of binanceus were purged + # Also check tables were properly migrated and we lost no data trades_query = cursor.execute( 'SELECT id, time, location, base_asset, quote_asset, type, amount, rate, fee,' 'fee_currency, link, notes from trades ORDER BY time ASC;', ).fetchall() - assert trades_query == [('foo2', 1, 'B', 'ETH', 'BTC', 'A', '1', '1', '1', 'ETH', '', '')] + assert trades_query == [ + ('foo2', 1, 'B', 'ETH', 'BTC', 'A', '1', '1', '1', 'ETH', '', ''), + ('fac279d109466a908119816d2ee7af90fba28f1ae60437bcbfab10e40a62bbc7', 1493227049, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '867.46673624', '0.00114990', '0.00249374', 'BTC', 'l1', None), # noqa: E501 + ('56775b4b80f46d1dfc1b53fc0f6b61573c14142499bfe3a62e3371f7afc166db', 1493228252, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'A', '11348.12286689', '0.00008790', '0.00249374', 'BTC', 'l2', None), # noqa: E501 + ('d38687c92fd4b56f7241b38653390a72022709ef8835c289f6d49cc436b8e05a', 1498949799, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '150.00000000', '0.00091705', '0.00034389', 'BTC', 'l3', None), # noqa: E501 + ('cc31283b6e723f4a7364496b349869564b06b4320da3a14d95dcda1801369361', 1498966605, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '1000.00000000', '0.00091409', '0.00228521', 'BTC', 'l4', None), # noqa: E501 + ('c76a42bc8b32407b7444da89cc9f73c22fd6a1bc0055b7d4112e3e59709b2b5b', 1499011364, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '200.00000000', '0.00092401', '0.00046200', 'BTC', 'l5', None), # noqa: E501 + ('18810df423b24bb438360b23b50bd0fc11ed2d3701420651ef716f4840367894', 1499051024, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '120.00000000', '0.00091603', '0.00027480', 'BTC', 'l6', None), # noqa: E501 + ('27bb15dfc1a008c2efa2c5510b30808d8246ab903cf9f950ea7a175f0779925d', 1499675187, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '200.00000000', '0.00091747', '0.00045869', 'BTC', 'l7', None), # noqa: E501 + ('b0c4d1d816fa3448ba1ab1a285a35d71a42ff27d68438626eacecc4bf927ab07', 1499677638, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '1135.00000000', '0.00088101', '0.00249985', 'BTC', 'l8', None), # noqa: E501 + ('6b91fc81d40c121af7397c5f3351547de4bd3f288fe483ace14b7b7922cfcd36', 1500494329, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'B', '10490.34064784', '0.00024385', '0.00639521', 'BTC', 'l9', None), # noqa: E501 + ('59ea4f98a815467122d201d737b97f910c8918cfe8f476a74d51a4006ef1aaa8', 1500501003, 'D', '_ceth_0x08711D3B02C8758F2FB3ab4e80228418a7F8e39c', 'BTC', 'B', '857.78221905', '0.00018099', '0.00038812', 'BTC', 'l10', None), # noqa: E501 + ('dc4a8a1dd3ef78b6eae5ee69f24bed73196ba3677150015f80d28a70b963253c', 1501194075, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '510.21713000', '0.00039101', '0.00049875', 'BTC', 'l11', None), # noqa: E501 + ('8ba98869f1398d9d64d974a13bc1e686746c7068b765223a46f886ef9c100722', 1501585385, 'D', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', 'BTC', 'A', '731.28354007', '0.00034101', '0.00062343', 'BTC', 'l12', None), # noqa: E501 + ('6f583c1297a86beadb46e8876db9589bb2ca60c603a4eca77b331611a49cd482', 1599751485, 'A', 'ETH', 'EUR', 'A', '150', '210.15', '0.2', 'EUR', None, None), # noqa: E501 + ('3fc94b6b91de61d98b21bdba9a6e449b3ff25756f34a14d17dcf3979d08c4ee3', 1607172606, 'D', 'MAID', 'BTC', 'B', '15515.00000000', '0.00001299', '0.00040305', 'BTC', 'l13', None), # noqa: E501 + ('ecd64dba4367a42292988abb34ed46b3dda0d48728c629f9727706d198023d6c', 1610915040, 'A', 'ETH', 'EUR', 'A', '5', '0.1', '0.001', '_ceth_0x111111111117dC0aa78b770fA6A738034120C302', 'dsad', 'ads'), # noqa: E501c + ('7aae102d9240f7d5f7f0669d6eefb47f2d1cf5bba462b0cee267719e9272ffde', 1612302374, 'A', '_ceth_0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984', 'ETH', 'A', '1', '0.01241', '0', '_ceth_0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984', None, None), # noqa: E501 + ] asset_movements_query = cursor.execute('SELECT id, location, category, address, transaction_id, time, asset, amount, fee_asset, fee, link from asset_movements ORDER BY time ASC;').fetchall() # noqa: E501 - assert asset_movements_query == [('foo2', 'B', 'B', '0xaddy', '0xtxid', 1, 'YFI', '1', 'GNO', '1', 'customlink')] # noqa: E501 + assert asset_movements_query == [ + ('foo2', 'B', 'B', '0xaddy', '0xtxid', 1, 'ETH', '1', '_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F', '1', 'customlink'), # noqa: E501 + ('822511b6035c5d2a7a7ff82c21b61381016e76764e84f656aedcfbc3b7a2e2f4', 'L', 'B', '0xaddy', '0xtxid', 1, '_ceth_0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e', '1', '_ceth_0x6810e776880C02933D47DB1b9fc05908e5386b96', '1', 'customlink'), # noqa: E501 + ('98c8378892955d3c95cc24277188ad33504d2e668441ba23a270b6c65f00be43', 'D', 'A', '0xaddress', '0xtxid', 1493226738, 'BTC', '2.00000000', 'BTC', '0', 'link3'), # noqa: E501 + ('9713d2c2f90edfc375bfea1d014599e9f3a20eded94625c0a2483c4ab2692ff9', 'D', 'A', '0xaddress', '0xtxid', 1498941726, 'BTC', '4.20000000', 'BTC', '0', 'link2'), # noqa: E501 + ('86f6cda4bcd36e2fd0e8938fd3b31ebe895af2df6d8b60479c401cd846a3ccf8', 'D', 'B', '0xaddress', '0xtxid', 1501161076, 'BTC', '3.91944853', 'BTC', '0.00100000', 'link5'), # noqa: E501 + ('9a3ab62aea2892e9000c868ce29a471e34f57d3bbae7691b920bcf58fbea10ce', 'D', 'A', '0xaddress', '0xtxid', 1577666912, 'MAID', '15515.00000000', 'MAID', '0', 'link1'), # noqa: E501 + ('79d6d91d1fd2acf02a9d244e33ff340c04a938faaf0d1ba10aba9d8ae55b11cc', 'D', 'B', '0xaddress', '0xtxid', 1607094370, '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', '4753.96740631', '_ceth_0xB9e7F8568e08d5659f5D29C4997173d84CdF2607', '160.00000000', 'link4'), # noqa: E501 + ] # noqa: E501 + query = cursor.execute('SELECT identifier, timestamp, type, location, amount, asset, rate, rate_asset, link, notes from ledger_actions;') # noqa: E501 + assert query.fetchall() == [ + (1, 1611260690, 'A', 'A', '1', '_ceth_0x0E8d6b471e332F140e7d9dbB99E5E3822F728DA6', None, None, None, None), # noqa: E501 + (2, 1610483475, 'A', 'A', '1', '_ceth_0xB6eD7644C69416d67B522e20bC294A9a9B405B31', None, None, 'sad', 'asdsad'), # noqa: E501 + ] + # Check manually tracked balances after upgrades + query = cursor.execute('SELECT asset, label, amount, location from manually_tracked_balances;') # noqa: E501 + assert query.fetchall() == [ + ('EUR', 'test eur balance', '1', 'A'), + ('USD', 'test usd balance', '1', 'A'), + ('CNY', 'test CNY balance', '1', 'A'), + ('_ceth_0x8Ab7404063Ec4DBcfd4598215992DC3F8EC853d7', 'exotic asset', '1500', 'A'), + ('_ceth_0x111111111117dC0aa78b770fA6A738034120C302', 'test for duplication', '100000', 'J'), # noqa: E501 + ('_ceth_0x48Fb253446873234F2fEBbF9BdeAA72d9d387f94', 'test custom token balance', '65', 'A'), # noqa: E501 + ('_ceth_0x50D1c9771902476076eCFc8B2A83Ad6b9355a4c9', 'test_asset_with_same_symbol', '85', 'A'), # noqa: E501 + ] + # Check margin positions after upgrade + query = cursor.execute('SELECT id, location, open_time, close_time, profit_loss, pl_currency, fee, fee_currency, link, notes from margin_positions;') # noqa: E501 + raw_upgraded = query.fetchall() + assert raw_upgraded == [ + ("3ebd1ff33f6b6431778db56393e6105b94b0b23d0976462d70279e6f82db9924", "C", 1, 5, "500", "ETH", "1", "_ceth_0x6810e776880C02933D47DB1b9fc05908e5386b96", "", ""), # noqa: E501 + ("2ecdf50622f0ad6277b2c4b28954118753db195c9ae2005ce7da7b30d4a873c4", "A", 1, 5, "1", "BTC", "1", "_ceth_0x255Aa6DF07540Cb5d3d297f0D0D4D84cb52bc8e6", "", ""), # noqa: E501 + ("bec34827cd9ce879e91d45dfe11942752f810504439701ff7f3d005850f458a8", "A", 0, 5, "1", "_ceth_0x0bc529c00C6401aEF6D220BE8C6Ea1667F6Ad93e", "1", "_ceth_0x1494CA1F11D487c2bBe4543E90080AeBa4BA3C2b", "", ""), # noqa: E501 + ] used_query_ranges = cursor.execute('SELECT * from used_query_ranges').fetchall() assert used_query_ranges == [ ('kraken_trades', 0, 1), ('kraken_asset_movements', 0, 1), ] + # Check the tables that should have had their data deleted had them + assert cursor.execute('SELECT COUNT(*) from adex_events;').fetchone()[0] == 0 + assert cursor.execute('SELECT COUNT(*) from aave_events;').fetchone()[0] == 0 + assert cursor.execute('SELECT COUNT(*) from yearn_vaults_events;').fetchone()[0] == 0 + assert cursor.execute('SELECT COUNT(*) from ethereum_accounts_details;').fetchone()[0] == 0 + # check that the timed balances are still there and are correct. + # 3 balances of unknown tokens should be gone + assert cursor.execute('SELECT COUNT(*) from timed_balances;').fetchone()[0] == 389 + query = cursor.execute('SELECT category, time, currency, amount, usd_value from timed_balances;') # noqa: E501 + for idx, entry in enumerate(query): + # the test DB also has some custom tokens which are not in the globalDB as of this writing + # 1st one is random fake address, 2nd one is vBNT + if entry[2] in ('_ceth_0xdb89d55d8878680FED2233ea6E1Ae7DF79C7073e', '_ceth_0x48Fb253446873234F2fEBbF9BdeAA72d9d387f94'): # noqa: E501 + continue + + # make sure the asset is understood + _ = Asset(entry[2]) + # check some specific entries + if idx == 388: + assert entry == ( + 'A', + 1616766011, + '_ceth_0x50D1c9771902476076eCFc8B2A83Ad6b9355a4c9', + '85', + '2909.55', + ) + elif idx == 384: + assert entry == ( + 'A', + 1616766011, + '_ceth_0x8Ab7404063Ec4DBcfd4598215992DC3F8EC853d7', + '1500', + '76.986000', + ) + elif idx == 2: + assert entry == ( + 'A', + 1610559319, + '_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F', + '90.5639', + '90.54578722', + ) + elif idx == 1: + assert entry == ( + 'A', + 1610559319, + '_ceth_0x4DC3643DbC642b72C158E7F3d2ff232df61cb6CE', + '0.1', + '0.001504', + ) + userdb_assets_num = cursor.execute('SELECT COUNT(*) from assets;').fetchone()[0] + globaldb_cursor = globaldb._conn.cursor() + globaldb_assets_num = globaldb_cursor.execute('SELECT COUNT(*) from assets;').fetchone()[0] + assert globaldb_assets_num == userdb_assets_num # Finally also make sure that we have updated to the target version assert db.get_version() == 26 diff --git a/rotkehlchen/tests/utils/database.py b/rotkehlchen/tests/utils/database.py index 90f538844f..14d3bbc879 100644 --- a/rotkehlchen/tests/utils/database.py +++ b/rotkehlchen/tests/utils/database.py @@ -98,3 +98,11 @@ def mock_dbhandler_update_owned_assets() -> _patch: 'rotkehlchen.db.dbhandler.DBHandler.update_owned_assets_in_globaldb', lambda x: None, ) + + +def mock_dbhandler_add_globaldb_assetids() -> _patch: + """Just make sure add globalds assetids does nothing for older DB tests""" + return patch( + 'rotkehlchen.db.dbhandler.DBHandler.add_globaldb_assetids', + lambda x: None, + ) diff --git a/rotkehlchen/tests/utils/premium.py b/rotkehlchen/tests/utils/premium.py index f353ef0d4a..3263c80e9b 100644 --- a/rotkehlchen/tests/utils/premium.py +++ b/rotkehlchen/tests/utils/premium.py @@ -15,7 +15,7 @@ from rotkehlchen.tests.utils.mock import MockResponse # Probably invalid remote data but bigger than the test local DB -INVALID_BUT_BIGGER_REMOTE_DATA = base64.b64encode(make_random_b64bytes(16048)).decode() +INVALID_BUT_BIGGER_REMOTE_DATA = base64.b64encode(make_random_b64bytes(86048)).decode() # Remote data containing a different main currency (GBP) REMOTE_DATA = b'faIQsH8WVnRtaDinjp7wyM22/Q7uM8scZ5C/ZEZZFpS/AUb10oc+HjDuyD5sQKkwTBmlfM4SSjooCXesURz2i1/YyhYjWK2YIECSdPhjUYrdvzNWLFmTtCu+lEw7urhYkC+22dpld3CO94ZM3xXr7WHzul+QEO9mT+snPGfKLAcSJhBSK/GXFi37Apgiz7u7LbgJ/Wp1D5C8ibYDIr83c39kaKvlDH6C+p7g9ncmy5meaa4vePxJlZZV6ay3x+yjZdffTdGm0RNl8OFzBNGs+eREjqk+vfWul5PgfHLdztaDGuYyhiw2+Hcos59BNpv3SHz9rBe6vcE7BZYPA+9zS77+qcQEyy9GEGXgCPX2UJpKNDR3ykHZtY+e2T+koBnzcspqEb9cDRe8pURWXcgNPh2+VSz4tDHF+GFEqbostAyZhkwjlB39QiJkK/HVymfLWtWL+/lQcvIrMlK6IVWmuEDCE74q0/RQX+uh6r3O+OTR5x+9FZEB3RcOhmx8OuLRCLptUL+KDqpINLeaM2Kl0TQjRj9Ot/8Q2iMe0wvzEKPj0c/8zJ8gJbEgrNS5xgWS8wtgiBdGlFYq/iopVcd8Gr1oZkK9RNPCKGXArwZ6cakMmf9nYceLOjaaw6FIV9445aRuc2n67cR96/1+697HC+x012va6akjf6ExHPjricKqlOfJhZebXGiPwXwH4m8S2aFSx7yCB8KzLNGWhgm1r7e0o75lBaCi0WONVMxym+PhfwYvZwrhmGlVqCIfucHaXYvwFBMawFBHH8LThpOg2GbKK3nLU/JXejuAK6xVWLgWt8wAWNJk3ZMajLkEhtxYtQAvggwCyvc3JQCbdQo1oAQA0dfTfqAJ54QQ5NJ6KI2G7fJSEUOIZnV+dcwjLLe6axIgj6JabV5xyguVH6jGmyN6ncbMY7gproA+jSNeZFPw+RQTHccrlsjPtDfF4HLHxiNKISrded7FIR5VGSqO6Gn1ML4l0Fst8kVxZuUpFzsXtgdQ8/DBjBAmGrjs+iDwJbh24J1jwZzukYuGu+Pibj68YET4eon2uENjqBGDekbFoGOKpimAXw5mIkZ/m5JWeGuhSqHUk7jSLHP96MBGMgh92SPLcucECgq7jGx0uCpzM6fAwAdoAQSOzdL5GmklNz9EYb5vqULAT6J2Ipdxe5nzE0fHne214B8PuGC5+uPU7NJLWl/6ubMaPLK4wSjPmj03m74FDcPVw5Kf0EdMt+9bHnLFtQY5a0XlopYU7xfBGbxoEmG2ixVoUNJvZD6tDRBp8X+yhPVZIWXBqx9QrsHFlYg/OFOsuVzx8OfmmPyj8IKAZKFNPElMhKgyfS/I2etkTqLc08vE0pNb4q2JmOh6Dh1eelJfpKujdxcYIT6MPGQYP6r30PFLnGWt+aNEJRMMFpK/R/iRIqYX6vMcgJHacOOz7QiPOcm9zutIOQx6vCqM1wFgxuR7Gd4lifwwf6VSyXK9W7RQ/9O8MQFoW4JEaLjcu5Axffw7rZpdG/WbQi6l+W5eX1m1ijPDz+/NkFbv6j6ZFUEwqJgegKDc5KUP7KvBk5T7F8EoWgpxXWKZl2XWhv6DZudRPkj3xyY8yLu5gCcvy/oHTU6X7ILh7mgxX2dl0IgFuCbaGHq7eIXPOCzist55aBL3haBQegQEG3xTMzX4zK+v+CWi6cTr+iDqWyVWgy61ZnB5t5+HrQ9VHfJALQhb00Qh/Jj5Vt0VIvFfqXKGazEksSdLY4HzxmckJIPMpSiRCglKYwC2Tc/YXZs5IaobSaEGcHJhkdkcON2CJMOfA4YXylHh4fQMZEGXeR0SQnsgcydpEj1xzY5Ucz0su8mEUWNaGH/PwatuVIOWk59ICrTfy9iL2yqnUxmFIdixUZiHnIihlHGOoskdOsor7akrRc7Ed0ztHtJxmWm/MwjbwYiXWYAiMLngEKJb96usa01uslyAIwylKa7mRjlj6k+sZ4NtkRS4e4Kdg9rvK9DR7uFuP+iFcjE1Xk12ANWFJi6BM51dmySjnep95Ur91cJXKPD3PYezSDUmjj3MSWtVMrSnOmbQuORoDmEW9Evp/D+nRsbiz32Kt/PLskNhrYbSoUlToaVWdzc0fUosZZ6kH+CnF7GbFNZU2Huc0qQsxesDsb2WbrXlg0Bidj43q7Ha3ZYJxTFTj3xZpBMYR9lhVPjdPb4Hl2ibtzwWw87fN6EuvyAu+4Sfujn4zsMu4qXA5YA1gStAaqlmZFa+0jKOrswCu0lC7h6i9FgSTpemUTWaMsaksk5yVRoBUGPM4AQXapSTjHP/sz+5kHlUPsEvaa68Ko+MwtxPFlrwzL4o6fOuLHR96q5lznuHrz26FihWfdeALNAUP2JA/d5W2TsEpdPCObuQcSZicONk4fnXmISWdVQQs4E/Oq/aQESIL55kO6GHEvKGzIYr9gGX3z+xTJoJg1fLTsiAWoyp1pCLOGP3jdsQSAo9MQSzHd6V8l5MRjgtCtOP4f2SarSi+QBEySiL0Vm/sF7sb2jcQu8Kc/wcoE8XQAujkyXJLoNJreTPaUs852Q/PRhOuTomMtNCbnR+L3jWEIzsM4w+HbGTPJQWJot3tjtNk4Zb3CmT+DuBlCnFs1kKXtwDHKReiohtt/pLA0pw4qzyMCbb8eyz0om/ik/o+CGL7QfpoeQBg1/LCcVPERBALETSoK78eM04YOc8LR5FiksJDPjw85bA2a9rIp47rbN+jBR7lkhCsCHhGw3mToLU/rP8Z+95Habk6SfnN1KasWDxB4Zzfx3OXxHmQQNK+NepDm/T3UtBA4eR9o9x3USgpy7+joQJRAxM4+dql/r82ULtWdmGxCabWwlzszOJFZ2wC6jPBACngADRQz1zuwDSlXo4so/k/UARASknsdX2Mz0F2mp1WHym+05ZA3X9NxIwm7UCmxuEI6KTRZKsHuuPL6dpzbo07c5vTFfrCLO/Lql9y8U5gwFIi3Eu7dRwm5i2U5s4xzaOjbsRNkFtDJauG1RitQuZutfUEIQD1AgjdZ65FeL21SWQKvZZXGL7SXhTGiiDXzWoKelFjbI0VEM+QrdjYmXq0UbnI7JOR05qTzsGgOMJ6mhqHH4T6gDOjn6RQuA=' # noqa: E501 # Remote data containing an older DB version @@ -208,9 +208,12 @@ def assert_db_got_replaced(rotkehlchen_instance: Rotkehlchen, username: str): assert rotkehlchen_instance.data.db.get_main_currency() == A_GBP # Also check a copy of our old DB is kept around. directory = os.path.join(rotkehlchen_instance.data.data_directory, username) - files = [os.path.join(directory, f) for f in os.listdir(directory)] + files = [ + os.path.join(directory, f) for f in os.listdir(directory) + if not f.endswith('backup') or f.startswith('rotkehlchen_db') + ] msg = f'Expected 2 or 3 files in the directory but got {files}' - assert len(files) in (2, 3, 4), msg # 3rd file is the dbinfo.json, 4th v24 DB ackup + assert len(files) in (2, 3), msg # 3rd file is the dbinfo.json # The order of the files is not guaranteed main_db_exists = False backup_db_exists = False diff --git a/rotkehlchen/tests/utils/rotkehlchen.py b/rotkehlchen/tests/utils/rotkehlchen.py index 965606f32a..a2b957e4a5 100644 --- a/rotkehlchen/tests/utils/rotkehlchen.py +++ b/rotkehlchen/tests/utils/rotkehlchen.py @@ -209,14 +209,6 @@ def add_starting_balances(datahandler) -> List[DBAssetBalance]: ), ] datahandler.db.add_multiple_balances(balances) - # Also add an unknown/invalid asset. This will generate a warning - cursor = datahandler.db.conn.cursor() - cursor.execute( - 'INSERT INTO timed_balances(' - ' time, currency, amount, usd_value) ' - ' VALUES(?, ?, ?, ?)', - (1469326500, 'ADSADX', '10.1', '100.5'), - ) datahandler.db.conn.commit() location_data = [