Merge pull request #2938 from LefterisJP/workon_2906

Implement asset constraints in the user DB

rotkehlchen/api/rest.py

@@ -48,7 +48,7 @@
 from rotkehlchen.chain.ethereum.typing import CustomEthereumToken
 from rotkehlchen.constants.assets import A_ETH
 from rotkehlchen.constants.misc import ZERO
-from rotkehlchen.constants.resolver import ETHEREUM_DIRECTIVE
+from rotkehlchen.constants.resolver import ethaddress_to_identifier
 from rotkehlchen.db.ledger_actions import DBLedgerActions
 from rotkehlchen.db.queried_addresses import QueriedAddresses
 from rotkehlchen.db.settings import ModifiableDBSettings
@@ -1311,8 +1311,8 @@ def get_asset_types() -> Response:
         types = [str(x) for x in AssetType]
         return api_response(_wrap_in_ok_result(types), status_code=HTTPStatus.OK)
 
-    @staticmethod
-    def add_custom_asset(asset_type: AssetType, **kwargs: Any) -> Response:
+    @require_loggedin_user()
+    def add_custom_asset(self, asset_type: AssetType, **kwargs: Any) -> Response:
         globaldb = GlobalDBHandler()
         # There is no good way to figure out if an asset already exists in the DB
         # Best approximation we can do is this.
@@ -1341,6 +1341,7 @@ def add_custom_asset(asset_type: AssetType, **kwargs: Any) -> Response:
         except InputError as e:
             return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT)
 
+        self.rotkehlchen.data.db.add_asset_identifiers([asset_id])
         return api_response(
             _wrap_in_ok_result({'identifier': asset_id}),
             status_code=HTTPStatus.OK,
@@ -1362,6 +1363,7 @@ def delete_custom_asset(self, identifier: str) -> Response:
         # Before deleting, also make sure we have up to date global DB owned data
         self.rotkehlchen.data.db.update_owned_assets_in_globaldb()
         try:
+            self.rotkehlchen.data.db.delete_asset_identifier(identifier)
             GlobalDBHandler().delete_custom_asset(identifier)
         except InputError as e:
             return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT)
@@ -1391,10 +1393,9 @@ def get_custom_ethereum_tokens(address: Optional[ChecksumEthAddress]) -> Respons
             log_result=False,
         )
 
-    @staticmethod
-    def add_custom_ethereum_token(token: CustomEthereumToken) -> Response:
-        # TODO: hacky. Clean this up when we allow addition of all assets
-        identifier = ETHEREUM_DIRECTIVE + token.address
+    @require_loggedin_user()
+    def add_custom_ethereum_token(self, token: CustomEthereumToken) -> Response:
+        identifier = ethaddress_to_identifier(token.address)
         try:
             GlobalDBHandler().add_asset(
                 asset_id=identifier,
@@ -1404,6 +1405,7 @@ def add_custom_ethereum_token(token: CustomEthereumToken) -> Response:
         except InputError as e:
             return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT)
 
+        self.rotkehlchen.data.db.add_asset_identifiers([identifier])
         return api_response(
             _wrap_in_ok_result({'identifier': identifier}),
             status_code=HTTPStatus.OK,
@@ -1428,7 +1430,9 @@ def edit_custom_ethereum_token(token: CustomEthereumToken) -> Response:
     def delete_custom_ethereum_token(self, address: ChecksumEthAddress) -> Response:
         # Before deleting, also make sure we have up to date global DB owned data
         self.rotkehlchen.data.db.update_owned_assets_in_globaldb()
+
         try:
+            self.rotkehlchen.data.db.delete_asset_identifier(ethaddress_to_identifier(address))
             identifier = GlobalDBHandler().delete_ethereum_token(address=address)
         except InputError as e:
             return api_response(wrap_in_fail_result(str(e)), status_code=HTTPStatus.CONFLICT)
@@ -2959,6 +2963,7 @@ def _perform_assets_updates(
             return {'result': None, 'message': str(e), 'status_code': HTTPStatus.BAD_GATEWAY}
 
         if result is None:
+            self.rotkehlchen.data.db.add_globaldb_assetids()
             return OK_RESULT
 
         return {

rotkehlchen/db/dbhandler.py

@@ -249,6 +249,7 @@ def __init__(
         if initial_settings is not None:
             self.set_settings(initial_settings)
         self.update_owned_assets_in_globaldb()
+        self.add_globaldb_assetids()
 
     def __del__(self) -> None:
         if hasattr(self, 'conn') and self.conn:
@@ -750,8 +751,9 @@ def add_multiple_balances(self, balances: List[DBAssetBalance]) -> None:
                 )
             except sqlcipher.IntegrityError:  # pylint: disable=no-member
                 self.msg_aggregator.add_warning(
-                    f'Tried to add a timed_balance for {entry.asset.identifier} at'
-                    f' already existing timestamp {entry.time}. Skipping.',
+                    f'Adding timed_balance failed. Either asset with identifier '
+                    f'{entry.asset.identifier} is not known or an entry for timestamp '
+                    f'{entry.time} already exists. Skipping.',
                 )
                 continue
         self.conn.commit()
@@ -3243,6 +3245,43 @@ def update_owned_assets_in_globaldb(self) -> None:
         assets = self.query_owned_assets()
         GlobalDBHandler().add_user_owned_assets(assets)
 
+    def add_asset_identifiers(self, asset_identifiers: List[str]) -> None:
+        """Adds an asset to the user db asset identifier table"""
+        cursor = self.conn.cursor()
+        cursor.executemany(
+            'INSERT OR IGNORE INTO assets(identifier) VALUES(?);',
+            [(x,) for x in asset_identifiers],
+        )
+        self.conn.commit()
+        self.update_last_write()
+
+    def add_globaldb_assetids(self) -> None:
+        """Makes sure that all the GlobalDB asset identifiers are mirrored in the user DB"""
+        cursor = GlobalDBHandler()._conn.cursor()  # after succesfull update add all asset ids
+        query = cursor.execute('SELECT identifier from assets;')
+        self.add_asset_identifiers([x[0] for x in query])
+
+    def delete_asset_identifier(self, asset_id: str) -> None:
+        """Deletes an asset identifier from the user db asset identifier table
+
+        May raise:
+        - InputError if a foreign key error is encountered during deletion
+        """
+        cursor = self.conn.cursor()
+        try:
+            cursor.execute(
+                'DELETE FROM assets WHERE identifier=?;',
+                (asset_id,),
+            )
+        except sqlcipher.IntegrityError as e:  # pylint: disable=no-member
+            raise InputError(
+                f'Failed to delete asset with id {asset_id} from the DB since '
+                f'the user owns it now or did some time in the past',
+            ) from e
+
+        self.conn.commit()
+        self.update_last_write()
+
     def get_latest_location_value_distribution(self) -> List[LocationData]:
         """Gets the latest location data
 

rotkehlchen/db/schema.py

@@ -142,6 +142,12 @@
 INSERT OR IGNORE INTO action_type(type, seq) VALUES ('D', 4);
 """
 
+DB_CREATE_ASSETS = """
+CREATE TABLE IF NOT EXISTS assets (
+    identifier TEXT NOT NULL PRIMARY KEY
+);
+"""
+
 DB_CREATE_IGNORED_ACTIONS = """
 CREATE TABLE IF NOT EXISTS ignored_actions (
     type CHAR(1) NOT NULL DEFAULT('A') REFERENCES action_type(type),
@@ -154,9 +160,10 @@
 CREATE TABLE IF NOT EXISTS timed_balances (
     category CHAR(1) NOT NULL DEFAULT('A') REFERENCES balance_category(category),
     time INTEGER,
-    currency VARCHAR[12],
+    currency TEXT,
     amount TEXT,
     usd_value TEXT,
+    FOREIGN KEY(currency) REFERENCES assets(identifier) ON UPDATE CASCADE,
     PRIMARY KEY (time, currency, category)
 );
 """
@@ -200,13 +207,15 @@
     timestamp INTEGER NOT NULL,
     tx_hash VARCHAR[66] NOT NULL,
     log_index INTEGER NOT NULL,
-    asset1 VARCHAR[12] NOT NULL,
+    asset1 TEXT NOT NULL,
     asset1_amount TEXT NOT NULL,
     asset1_usd_value TEXT NOT NULL,
-    asset2 VARCHAR[12],
+    asset2 TEXT,
     asset2amount_borrowrate_feeamount TEXT,
     asset2usd_value_accruedinterest_feeusdvalue TEXT,
     borrow_rate_mode VARCHAR[10],
+    FOREIGN KEY(asset1) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(asset2) REFERENCES assets(identifier) ON UPDATE CASCADE,
     PRIMARY KEY (event_type, tx_hash, log_index)
 );
 """
@@ -215,10 +224,10 @@
 CREATE TABLE IF NOT EXISTS yearn_vaults_events (
     address VARCHAR[42] NOT NULL,
     event_type VARCHAR[10] NOT NULL,
-    from_asset VARCHAR[44] NOT NULL,
+    from_asset TEXT NOT NULL,
     from_amount TEXT NOT NULL,
     from_usd_value TEXT NOT NULL,
-    to_asset VARCHAR[44] NOT NULL,
+    to_asset TEXT NOT NULL,
     to_amount TEXT NOT NULL,
     to_usd_value TEXT NOT NULL,
     pnl_amount TEXT,
@@ -227,6 +236,8 @@
     timestamp INTEGER NOT NULL,
     tx_hash VARCHAR[66] NOT NULL,
     log_index INTEGER NOT NULL,
+    FOREIGN KEY(from_asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(to_asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
     PRIMARY KEY (event_type, tx_hash, log_index)
 );
 """
@@ -287,10 +298,11 @@
 
 DB_CREATE_MANUALLY_TRACKED_BALANCES = """
 CREATE TABLE IF NOT EXISTS manually_tracked_balances (
-    asset VARCHAR[24] NOT NULL,
+    asset TEXT NOT NULL,
     label TEXT NOT NULL PRIMARY KEY,
     amount TEXT,
-    location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location)
+    location CHAR(1) NOT NULL DEFAULT('A') REFERENCES location(location),
+    FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE
 );
 """
 
@@ -324,9 +336,12 @@
     amount TEXT NOT NULL,
     rate TEXT NOT NULL,
     fee TEXT,
-    fee_currency TEXT[10],
+    fee_currency TEXT,
     link TEXT,
-    notes TEXT
+    notes TEXT,
+    FOREIGN KEY(base_asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(quote_asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE
 );
 """
 
@@ -337,11 +352,13 @@
     open_time INTEGER,
     close_time INTEGER,
     profit_loss TEXT,
-    pl_currency VARCHAR[10],
+    pl_currency TEXT NOT NULL,
     fee TEXT,
-    fee_currency VARCHAR[10],
+    fee_currency TEXT,
     link TEXT,
-    notes TEXT
+    notes TEXT,
+    FOREIGN KEY(pl_currency) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(fee_currency) REFERENCES assets(identifier) ON UPDATE CASCADE
 );
 """
 
@@ -353,11 +370,13 @@
     address TEXT,
     transaction_id TEXT,
     time INTEGER,
-    asset VARCHAR[10],
+    asset TEXT NOT NULL,
     amount TEXT,
-    fee_asset VARCHAR[10],
+    fee_asset TEXT,
     fee TEXT,
-    link TEXT
+    link TEXT,
+    FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(fee_asset) REFERENCES assets(identifier) ON UPDATE CASCADE
 );
 """
 
@@ -372,7 +391,9 @@
     rate TEXT,
     rate_asset TEXT,
     link TEXT,
-    notes TEXT
+    notes TEXT,
+    FOREIGN KEY(asset) REFERENCES assets(identifier) ON UPDATE CASCADE,
+    FOREIGN KEY(rate_asset) REFERENCES assets(identifier) ON UPDATE CASCADE
 );
 """
 
@@ -517,6 +538,7 @@
     channel_id TEXT,
     token TEXT,
     log_index INTEGER,
+    FOREIGN KEY(token) REFERENCES assets(identifier) ON UPDATE CASCADE,
     PRIMARY KEY (tx_hash, address, type, log_index)
 );
 """
@@ -603,14 +625,15 @@
 DB_SCRIPT_CREATE_TABLES = """
 PRAGMA foreign_keys=off;
 BEGIN TRANSACTION;
-{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}
+{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}{}
 COMMIT;
 PRAGMA foreign_keys=on;
 """.format(
     DB_CREATE_TRADE_TYPE,
     DB_CREATE_LOCATION,
     DB_CREATE_ASSET_MOVEMENT_CATEGORY,
     DB_CREATE_BALANCE_CATEGORY,
+    DB_CREATE_ASSETS,
     DB_CREATE_TIMED_BALANCES,
     DB_CREATE_TIMED_LOCATION_DATA,
     DB_CREATE_USER_CREDENTIALS,

rotkehlchen/db/upgrade_manager.py

@@ -262,7 +262,7 @@ def _perform_single_upgrade(self, upgrade: UpgradeRecord) -> None:
                 raise DBUpgradeError(error_message) from e
 
             # for some upgrades even for success keep the backup of the previous db
-            if upgrade.from_version == 24:
+            if upgrade.from_version in (24, 25):
                 shutil.copyfile(
                     tmp_db_path,
                     os.path.join(self.db.user_data_dir, tmp_db_filename),