Fix tests after assets identifier in user DB changes

rotkehlchen/api/rest.py

@@ -1311,6 +1311,7 @@ def get_asset_types() -> Response:
         types = [str(x) for x in AssetType]
         return api_response(_wrap_in_ok_result(types), status_code=HTTPStatus.OK)
 
+    @require_loggedin_user()
     def add_custom_asset(self, asset_type: AssetType, **kwargs: Any) -> Response:
         globaldb = GlobalDBHandler()
         # There is no good way to figure out if an asset already exists in the DB
@@ -1392,6 +1393,7 @@ def get_custom_ethereum_tokens(address: Optional[ChecksumEthAddress]) -> Respons
             log_result=False,
         )
 
+    @require_loggedin_user()
     def add_custom_ethereum_token(self, token: CustomEthereumToken) -> Response:
         identifier = ethaddress_to_identifier(token.address)
         try:
@@ -2961,9 +2963,7 @@ def _perform_assets_updates(
             return {'result': None, 'message': str(e), 'status_code': HTTPStatus.BAD_GATEWAY}
 
         if result is None:
-            cursor = GlobalDBHandler()._conn.cursor()  # after succesfull update add all asset ids
-            query = cursor.execute('SELECT identifier from assets;')
-            self.rotkehlchen.data.db.add_asset_identifiers([x[0] for x in query])
+            self.rotkehlchen.data.db.add_globaldb_assetids()
             return OK_RESULT
 
         return {

rotkehlchen/db/dbhandler.py

@@ -249,6 +249,7 @@ def __init__(
         if initial_settings is not None:
             self.set_settings(initial_settings)
         self.update_owned_assets_in_globaldb()
+        self.add_globaldb_assetids()
 
     def __del__(self) -> None:
         if hasattr(self, 'conn') and self.conn:
@@ -750,8 +751,9 @@ def add_multiple_balances(self, balances: List[DBAssetBalance]) -> None:
                 )
             except sqlcipher.IntegrityError:  # pylint: disable=no-member
                 self.msg_aggregator.add_warning(
-                    f'Tried to add a timed_balance for {entry.asset.identifier} at'
-                    f' already existing timestamp {entry.time}. Skipping.',
+                    f'Adding timed_balance failed. Either asset with identifier '
+                    f'{entry.asset.identifier} is not known or an entry for timestamp '
+                    f'{entry.time} already exists. Skipping.',
                 )
                 continue
         self.conn.commit()
@@ -3253,6 +3255,12 @@ def add_asset_identifiers(self, asset_identifiers: List[str]) -> None:
         self.conn.commit()
         self.update_last_write()
 
+    def add_globaldb_assetids(self) -> None:
+        """Makes sure that all the GlobalDB asset identifiers are mirrored in the user DB"""
+        cursor = GlobalDBHandler()._conn.cursor()  # after succesfull update add all asset ids
+        query = cursor.execute('SELECT identifier from assets;')
+        self.add_asset_identifiers([x[0] for x in query])
+
     def delete_asset_identifier(self, asset_id: str) -> None:
         """Deletes an asset identifier from the user db asset identifier table
 
@@ -3267,7 +3275,8 @@ def delete_asset_identifier(self, asset_id: str) -> None:
             )
         except sqlcipher.IntegrityError as e:  # pylint: disable=no-member
             raise InputError(
-                f'Failed to delete asset with id {asset_id} from the DB due to {e}',
+                f'Failed to delete asset with id {asset_id} from the DB since '
+                f'the user owns it now or did some time in the past',
             ) from e
 
         self.conn.commit()

rotkehlchen/tests/api/test_custom_assets.py

@@ -16,7 +16,7 @@
 
 
 @pytest.mark.parametrize('use_clean_caching_directory', [True])
-@pytest.mark.parametrize('start_with_logged_in_user', [False])
+@pytest.mark.parametrize('start_with_logged_in_user', [True])
 def test_adding_custom_assets(rotkehlchen_api_server, globaldb):
     """Test that the endpoint for adding a custom asset works"""
 
@@ -136,7 +136,7 @@ def test_adding_custom_assets(rotkehlchen_api_server, globaldb):
 
 
 @pytest.mark.parametrize('use_clean_caching_directory', [True])
-@pytest.mark.parametrize('start_with_logged_in_user', [False])
+@pytest.mark.parametrize('start_with_logged_in_user', [True])
 def test_editing_custom_assets(rotkehlchen_api_server, globaldb):
     """Test that the endpoint for editing a custom asset works"""
 
@@ -417,10 +417,7 @@ def test_custom_asset_delete_guard(rotkehlchen_api_server):
         ),
         json={'identifier': custom1_id},
     )
-    expected_msg = (
-        'Tried to delete asset with name "foo token" and symbol "FOO" but its deletion '
-        'would violate a constraint'
-    )
+    expected_msg = 'Failed to delete asset with id'
     assert_error_response(
         response=response,
         contained_in_msg=expected_msg,

rotkehlchen/tests/api/test_custom_ethereum_tokens.py

@@ -92,7 +92,7 @@ def test_query_custom_tokens(rotkehlchen_api_server):
 
 
 @pytest.mark.parametrize('use_clean_caching_directory', [True])
-@pytest.mark.parametrize('start_with_logged_in_user', [False])
+@pytest.mark.parametrize('start_with_logged_in_user', [True])
 @pytest.mark.parametrize('custom_ethereum_tokens', [INITIAL_TOKENS])
 def test_adding_custom_tokens(rotkehlchen_api_server):
     """Test that the endpoint for adding a custom ethereum token works"""
@@ -233,7 +233,7 @@ def test_adding_custom_tokens(rotkehlchen_api_server):
 
 
 @pytest.mark.parametrize('use_clean_caching_directory', [True])
-@pytest.mark.parametrize('start_with_logged_in_user', [False])
+@pytest.mark.parametrize('start_with_logged_in_user', [True])
 @pytest.mark.parametrize('custom_ethereum_tokens', [INITIAL_TOKENS])
 def test_editing_custom_tokens(rotkehlchen_api_server):
     """Test that the endpoint for editing a custom ethereum token works"""
@@ -348,7 +348,7 @@ def test_deleting_custom_tokens(rotkehlchen_api_server):
     )
     expected_msg = (
         f'Tried to delete ethereum token with address {non_existing_address} '
-        f'but it was not found in the DB',
+        f'but it was not found in the DB'
     )
     assert_error_response(
         response=response,
@@ -462,10 +462,7 @@ def test_custom_tokens_delete_guard(rotkehlchen_api_server):
         ),
         json={'address': INITIAL_TOKENS[0].address},
     )
-    expected_msg = (
-        f'Tried to delete ethereum token with address {INITIAL_TOKENS[0].address} '
-        f'but its deletion would violate a constraint so deletion failed.'
-    )
+    expected_msg = 'Failed to delete asset with id'
     assert_error_response(
         response=response,
         contained_in_msg=expected_msg,

rotkehlchen/tests/db/test_db.py

@@ -86,6 +86,7 @@
 from rotkehlchen.utils.serialization import rlk_jsondumps
 
 TABLES_AT_INIT = [
+    'assets',
     'aave_events',
     'yearn_vaults_events',
     'timed_balances',
@@ -619,13 +620,6 @@ def test_query_owned_assets(data_dir, username):
         ),
     ])
     data.db.add_multiple_balances(balances)
-    cursor = data.db.conn.cursor()
-    cursor.execute(
-        'INSERT INTO timed_balances('
-        '    time, currency, amount, usd_value, category) '
-        ' VALUES(?, ?, ?, ?, ?)',
-        (1469326500, 'ADSADX', '10.1', '100.5', 'A'),
-    )
     data.db.conn.commit()
 
     # also make sure that assets from trades are included
@@ -697,8 +691,7 @@ def test_query_owned_assets(data_dir, username):
     assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH}  # noqa: E501
     assert all(isinstance(x, Asset) for x in assets_list)
     warnings = data.db.msg_aggregator.consume_warnings()
-    assert len(warnings) == 1
-    assert 'Unknown/unsupported asset ADSADX' in warnings[0]
+    assert len(warnings) == 0
 
 
 def test_get_latest_location_value_distribution(data_dir, username):

rotkehlchen/tests/db/test_db_upgrades.py

@@ -29,7 +29,10 @@
 )
 from rotkehlchen.db.upgrades.v13_v14 import REMOVED_ASSETS, REMOVED_ETH_TOKENS
 from rotkehlchen.errors import DBUpgradeError
-from rotkehlchen.tests.utils.database import mock_dbhandler_update_owned_assets
+from rotkehlchen.tests.utils.database import (
+    mock_dbhandler_add_globaldb_assetids,
+    mock_dbhandler_update_owned_assets,
+)
 from rotkehlchen.tests.utils.factories import make_ethereum_address
 from rotkehlchen.typing import ChecksumEthAddress
 from rotkehlchen.user_messages import MessagesAggregator
@@ -319,7 +322,7 @@ def test_upgrade_db_1_to_2(data_dir, username):
     ethereum accounts are now checksummed"""
     msg_aggregator = MessagesAggregator()
     data = DataHandler(data_dir, msg_aggregator)
-    with creation_patch, target_patch(1), mock_dbhandler_update_owned_assets():
+    with creation_patch, target_patch(1), mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids():  # noqa: E501
         data.unlock(username, '123', create_new=True)
     # Manually input a non checksummed account
     data.db.conn.commit()
@@ -348,14 +351,14 @@ def test_upgrade_db_1_to_2(data_dir, username):
 def test_upgrade_db_2_to_3(user_data_dir):
     """Test upgrading the DB from version 2 to version 3, rename BCHSV to BSV"""
     msg_aggregator = MessagesAggregator()
-    with creation_patch:
+    with creation_patch, mock_dbhandler_add_globaldb_assetids():
         db = _init_db_with_target_version(
             target_version=2,
             user_data_dir=user_data_dir,
             msg_aggregator=msg_aggregator,
         )
 
-    with mock_dbhandler_update_owned_assets():
+    with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids():
         populate_db_and_check_for_asset_renaming(
             db=db,
             user_data_dir=user_data_dir,
@@ -372,7 +375,7 @@ def test_upgrade_db_3_to_4(data_dir, username):
     the eth_rpc_port setting is changed to eth_rpc_endpoint"""
     msg_aggregator = MessagesAggregator()
     data = DataHandler(data_dir, msg_aggregator)
-    with creation_patch, target_patch(3), mock_dbhandler_update_owned_assets():
+    with creation_patch, target_patch(3), mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids():  # noqa: E501
         data.unlock(username, '123', create_new=True)
     # Manually set version and input the old rpcport setting
     cursor = data.db.conn.cursor()
@@ -387,7 +390,7 @@ def test_upgrade_db_3_to_4(data_dir, username):
     data.db.conn.commit()
 
     # now relogin and check that the setting has been changed and the version bumped
-    with mock_dbhandler_update_owned_assets():
+    with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids():
         del data
         data = DataHandler(data_dir, msg_aggregator)
         with target_patch(target_version=4):
@@ -409,14 +412,14 @@ def test_upgrade_db_3_to_4(data_dir, username):
 def test_upgrade_db_4_to_5(user_data_dir):
     """Test upgrading the DB from version 4 to version 5, rename BCC to BCH"""
     msg_aggregator = MessagesAggregator()
-    with creation_patch:
+    with creation_patch, mock_dbhandler_add_globaldb_assetids():
         db = _init_db_with_target_version(
             target_version=4,
             user_data_dir=user_data_dir,
             msg_aggregator=msg_aggregator,
         )
 
-    with mock_dbhandler_update_owned_assets():
+    with mock_dbhandler_update_owned_assets(), mock_dbhandler_add_globaldb_assetids():
         populate_db_and_check_for_asset_renaming(
             db=db,
             user_data_dir=user_data_dir,

rotkehlchen/tests/utils/database.py

@@ -98,3 +98,11 @@ def mock_dbhandler_update_owned_assets() -> _patch:
         'rotkehlchen.db.dbhandler.DBHandler.update_owned_assets_in_globaldb',
         lambda x: None,
     )
+
+
+def mock_dbhandler_add_globaldb_assetids() -> _patch:
+    """Just make sure add globalds assetids does nothing for older DB tests"""
+    return patch(
+        'rotkehlchen.db.dbhandler.DBHandler.add_globaldb_assetids',
+        lambda x: None,
+    )

rotkehlchen/tests/utils/premium.py

@@ -15,7 +15,7 @@
 from rotkehlchen.tests.utils.mock import MockResponse
 
 # Probably invalid remote data but bigger than the test local DB
-INVALID_BUT_BIGGER_REMOTE_DATA = base64.b64encode(make_random_b64bytes(16048)).decode()
+INVALID_BUT_BIGGER_REMOTE_DATA = base64.b64encode(make_random_b64bytes(86048)).decode()
 # Remote data containing a different main currency (GBP)
 REMOTE_DATA = b'faIQsH8WVnRtaDinjp7wyM22/Q7uM8scZ5C/ZEZZFpS/AUb10oc+HjDuyD5sQKkwTBmlfM4SSjooCXesURz2i1/YyhYjWK2YIECSdPhjUYrdvzNWLFmTtCu+lEw7urhYkC+22dpld3CO94ZM3xXr7WHzul+QEO9mT+snPGfKLAcSJhBSK/GXFi37Apgiz7u7LbgJ/Wp1D5C8ibYDIr83c39kaKvlDH6C+p7g9ncmy5meaa4vePxJlZZV6ay3x+yjZdffTdGm0RNl8OFzBNGs+eREjqk+vfWul5PgfHLdztaDGuYyhiw2+Hcos59BNpv3SHz9rBe6vcE7BZYPA+9zS77+qcQEyy9GEGXgCPX2UJpKNDR3ykHZtY+e2T+koBnzcspqEb9cDRe8pURWXcgNPh2+VSz4tDHF+GFEqbostAyZhkwjlB39QiJkK/HVymfLWtWL+/lQcvIrMlK6IVWmuEDCE74q0/RQX+uh6r3O+OTR5x+9FZEB3RcOhmx8OuLRCLptUL+KDqpINLeaM2Kl0TQjRj9Ot/8Q2iMe0wvzEKPj0c/8zJ8gJbEgrNS5xgWS8wtgiBdGlFYq/iopVcd8Gr1oZkK9RNPCKGXArwZ6cakMmf9nYceLOjaaw6FIV9445aRuc2n67cR96/1+697HC+x012va6akjf6ExHPjricKqlOfJhZebXGiPwXwH4m8S2aFSx7yCB8KzLNGWhgm1r7e0o75lBaCi0WONVMxym+PhfwYvZwrhmGlVqCIfucHaXYvwFBMawFBHH8LThpOg2GbKK3nLU/JXejuAK6xVWLgWt8wAWNJk3ZMajLkEhtxYtQAvggwCyvc3JQCbdQo1oAQA0dfTfqAJ54QQ5NJ6KI2G7fJSEUOIZnV+dcwjLLe6axIgj6JabV5xyguVH6jGmyN6ncbMY7gproA+jSNeZFPw+RQTHccrlsjPtDfF4HLHxiNKISrded7FIR5VGSqO6Gn1ML4l0Fst8kVxZuUpFzsXtgdQ8/DBjBAmGrjs+iDwJbh24J1jwZzukYuGu+Pibj68YET4eon2uENjqBGDekbFoGOKpimAXw5mIkZ/m5JWeGuhSqHUk7jSLHP96MBGMgh92SPLcucECgq7jGx0uCpzM6fAwAdoAQSOzdL5GmklNz9EYb5vqULAT6J2Ipdxe5nzE0fHne214B8PuGC5+uPU7NJLWl/6ubMaPLK4wSjPmj03m74FDcPVw5Kf0EdMt+9bHnLFtQY5a0XlopYU7xfBGbxoEmG2ixVoUNJvZD6tDRBp8X+yhPVZIWXBqx9QrsHFlYg/OFOsuVzx8OfmmPyj8IKAZKFNPElMhKgyfS/I2etkTqLc08vE0pNb4q2JmOh6Dh1eelJfpKujdxcYIT6MPGQYP6r30PFLnGWt+aNEJRMMFpK/R/iRIqYX6vMcgJHacOOz7QiPOcm9zutIOQx6vCqM1wFgxuR7Gd4lifwwf6VSyXK9W7RQ/9O8MQFoW4JEaLjcu5Axffw7rZpdG/WbQi6l+W5eX1m1ijPDz+/NkFbv6j6ZFUEwqJgegKDc5KUP7KvBk5T7F8EoWgpxXWKZl2XWhv6DZudRPkj3xyY8yLu5gCcvy/oHTU6X7ILh7mgxX2dl0IgFuCbaGHq7eIXPOCzist55aBL3haBQegQEG3xTMzX4zK+v+CWi6cTr+iDqWyVWgy61ZnB5t5+HrQ9VHfJALQhb00Qh/Jj5Vt0VIvFfqXKGazEksSdLY4HzxmckJIPMpSiRCglKYwC2Tc/YXZs5IaobSaEGcHJhkdkcON2CJMOfA4YXylHh4fQMZEGXeR0SQnsgcydpEj1xzY5Ucz0su8mEUWNaGH/PwatuVIOWk59ICrTfy9iL2yqnUxmFIdixUZiHnIihlHGOoskdOsor7akrRc7Ed0ztHtJxmWm/MwjbwYiXWYAiMLngEKJb96usa01uslyAIwylKa7mRjlj6k+sZ4NtkRS4e4Kdg9rvK9DR7uFuP+iFcjE1Xk12ANWFJi6BM51dmySjnep95Ur91cJXKPD3PYezSDUmjj3MSWtVMrSnOmbQuORoDmEW9Evp/D+nRsbiz32Kt/PLskNhrYbSoUlToaVWdzc0fUosZZ6kH+CnF7GbFNZU2Huc0qQsxesDsb2WbrXlg0Bidj43q7Ha3ZYJxTFTj3xZpBMYR9lhVPjdPb4Hl2ibtzwWw87fN6EuvyAu+4Sfujn4zsMu4qXA5YA1gStAaqlmZFa+0jKOrswCu0lC7h6i9FgSTpemUTWaMsaksk5yVRoBUGPM4AQXapSTjHP/sz+5kHlUPsEvaa68Ko+MwtxPFlrwzL4o6fOuLHR96q5lznuHrz26FihWfdeALNAUP2JA/d5W2TsEpdPCObuQcSZicONk4fnXmISWdVQQs4E/Oq/aQESIL55kO6GHEvKGzIYr9gGX3z+xTJoJg1fLTsiAWoyp1pCLOGP3jdsQSAo9MQSzHd6V8l5MRjgtCtOP4f2SarSi+QBEySiL0Vm/sF7sb2jcQu8Kc/wcoE8XQAujkyXJLoNJreTPaUs852Q/PRhOuTomMtNCbnR+L3jWEIzsM4w+HbGTPJQWJot3tjtNk4Zb3CmT+DuBlCnFs1kKXtwDHKReiohtt/pLA0pw4qzyMCbb8eyz0om/ik/o+CGL7QfpoeQBg1/LCcVPERBALETSoK78eM04YOc8LR5FiksJDPjw85bA2a9rIp47rbN+jBR7lkhCsCHhGw3mToLU/rP8Z+95Habk6SfnN1KasWDxB4Zzfx3OXxHmQQNK+NepDm/T3UtBA4eR9o9x3USgpy7+joQJRAxM4+dql/r82ULtWdmGxCabWwlzszOJFZ2wC6jPBACngADRQz1zuwDSlXo4so/k/UARASknsdX2Mz0F2mp1WHym+05ZA3X9NxIwm7UCmxuEI6KTRZKsHuuPL6dpzbo07c5vTFfrCLO/Lql9y8U5gwFIi3Eu7dRwm5i2U5s4xzaOjbsRNkFtDJauG1RitQuZutfUEIQD1AgjdZ65FeL21SWQKvZZXGL7SXhTGiiDXzWoKelFjbI0VEM+QrdjYmXq0UbnI7JOR05qTzsGgOMJ6mhqHH4T6gDOjn6RQuA='  # noqa: E501
 # Remote data containing an older DB version
@@ -208,9 +208,12 @@ def assert_db_got_replaced(rotkehlchen_instance: Rotkehlchen, username: str):
     assert rotkehlchen_instance.data.db.get_main_currency() == A_GBP
     # Also check a copy of our old DB is kept around.
     directory = os.path.join(rotkehlchen_instance.data.data_directory, username)
-    files = [os.path.join(directory, f) for f in os.listdir(directory)]
+    files = [
+        os.path.join(directory, f) for f in os.listdir(directory)
+        if not f.endswith('backup') or f.startswith('rotkehlchen_db')
+    ]
     msg = f'Expected 2 or 3 files in the directory but got {files}'
-    assert len(files) in (2, 3, 4), msg  # 3rd file is the dbinfo.json, 4th v24 DB ackup
+    assert len(files) in (2, 3), msg  # 3rd file is the dbinfo.json
     # The order of the files is not guaranteed
     main_db_exists = False
     backup_db_exists = False

rotkehlchen/tests/utils/rotkehlchen.py

@@ -209,14 +209,6 @@ def add_starting_balances(datahandler) -> List[DBAssetBalance]:
         ),
     ]
     datahandler.db.add_multiple_balances(balances)
-    # Also add an unknown/invalid asset. This will generate a warning
-    cursor = datahandler.db.conn.cursor()
-    cursor.execute(
-        'INSERT INTO timed_balances('
-        '    time, currency, amount, usd_value) '
-        ' VALUES(?, ?, ?, ?)',
-        (1469326500, 'ADSADX', '10.1', '100.5'),
-    )
     datahandler.db.conn.commit()
 
     location_data = [