Sigantures/Semantics bug fixes

ethtx/decoders/abi/abc.py

@@ -22,7 +22,7 @@
     TransactionMetadata,
     BlockMetadata,
 )
-from ethtx.providers.semantic_providers.semantics_repository import SemanticsRepository
+from ethtx.providers.semantic_providers import SemanticsRepository
 
 
 class ABIBasic:

ethtx/decoders/abi/decoder.py

@@ -206,7 +206,6 @@ def _decode_transaction(
             )
             raise e
 
-
         full_decoded_transaction.status = True
 
         return full_decoded_transaction

ethtx/decoders/abi/helpers/utils.py

@@ -18,9 +18,8 @@
     Signature,
     SignatureArg,
 )
-from ethtx.providers import FourByteProvider
-from ethtx.providers.semantic_providers.semantics_repository import SemanticsRepository
-from ethtx.providers.signature_provider import SignatureProvider
+from ethtx.providers.semantic_providers import SemanticsRepository
+from ethtx.providers.signature_provider import SignatureProvider, FourByteProvider
 
 log = logging.getLogger(__name__)
 
@@ -32,7 +31,7 @@ def decode_function_abi_with_external_source(
 ) -> Iterator[FunctionSemantics]:
     function = repository.get_most_used_signature(signature_hash=signature)
     if function:
-        log.info(
+        log.debug(
             "Successfully guessed function from SemanticsRepository - %s.",
             function.json(),
         )

ethtx/decoders/decoder_service.py

@@ -153,7 +153,9 @@ def decode_transaction(self, chain_id: str, tx_hash: str) -> DecodedTransaction:
 
         used_semantics = self.semantic_decoder.repository.end_record()
         log.info(
-            "Semantics used in decoding %s: %s", tx_hash, ", ".join(used_semantics)
+            "Semantics used in decoding %s: %s",
+            tx_hash,
+            ", ".join(used_semantics) if used_semantics else "",
         )
 
         return semantically_decoded_tx

ethtx/decoders/decoders/parameters.py

@@ -20,7 +20,6 @@
 
 
 def decode_event_parameters(data, topics, abi, anonymous):
-
     # making copy to avoid modifying of the original list
     amended_topics = topics.copy()
 
@@ -124,7 +123,6 @@ def decode_event_parameters(data, topics, abi, anonymous):
 def decode_function_parameters(
     input_data, output, abi, status=True, strip_signature=True
 ):
-
     if strip_signature and len(input_data) >= 10:
         stripped_input_data = input_data[10:]
     else:

ethtx/decoders/semantic/abc.py

@@ -22,7 +22,7 @@
     Proxy,
 )
 from ethtx.models.objects_model import BlockMetadata, TransactionMetadata
-from ethtx.providers.semantic_providers.semantics_repository import SemanticsRepository
+from ethtx.providers.semantic_providers import SemanticsRepository
 
 
 class SemanticSubmoduleAbc(ABC):

ethtx/decoders/semantic/balances.py

@@ -13,8 +13,8 @@
 from typing import List
 
 from ethtx.models.decoded_model import DecodedBalance, DecodedTransactionMetadata
-from .helpers.utils import get_badge
 from .abc import SemanticSubmoduleAbc
+from .helpers.utils import get_badge
 
 
 class SemanticBalancesDecoder(SemanticSubmoduleAbc):

ethtx/decoders/semantic/calls.py

@@ -13,8 +13,8 @@
 from typing import Dict
 
 from ethtx.models.decoded_model import DecodedCall, DecodedTransactionMetadata, Proxy
-from ethtx.semantics.standards.erc20 import ERC20_FUNCTIONS, ERC20_TRANSFORMATIONS
-from ethtx.semantics.standards.erc721 import ERC721_FUNCTIONS, ERC721_TRANSFORMATIONS
+from ethtx.semantics.standards.erc20 import ERC20_TRANSFORMATIONS
+from ethtx.semantics.standards.erc721 import ERC721_TRANSFORMATIONS
 from ethtx.utils.measurable import RecursionLimit
 from .abc import SemanticSubmoduleAbc
 from .helpers.utils import (

ethtx/decoders/semantic/helpers/utils.py

@@ -20,8 +20,8 @@
 
 from ethtx.decoders.decoders.parameters import decode_function_parameters
 from ethtx.models.decoded_model import AddressInfo
-from ethtx.semantics.utilities.functions import add_utils_to_context
 from ethtx.models.semantics_model import FunctionSemantics
+from ethtx.semantics.utilities.functions import add_utils_to_context
 
 log = logging.getLogger(__name__)
 
@@ -47,7 +47,6 @@ def get_eth_price() -> Optional[float]:
 
 
 def get_badge(address, sender, receiver):
-
     sender_address = sender.address if isinstance(sender, AddressInfo) else sender
     receiver_address = (
         receiver.address if isinstance(receiver, AddressInfo) else receiver

ethtx/decoders/semantic/transfers.py

@@ -13,17 +13,15 @@
 from typing import List
 
 from ethtx.models.decoded_model import DecodedTransfer, DecodedTransactionMetadata
-from .helpers.utils import get_badge
 from .abc import SemanticSubmoduleAbc
+from .helpers.utils import get_badge
 
 
 class SemanticTransfersDecoder(SemanticSubmoduleAbc):
     def decode(
         self, transfers: List[DecodedTransfer], tx_metadata: DecodedTransactionMetadata
     ) -> List[DecodedTransfer]:
-
         for transfer in transfers:
-
             # decode proper from and to addresses badges
             transfer.from_address.badge = get_badge(
                 transfer.from_address.address,

ethtx/ethtx.py

@@ -21,11 +21,11 @@
 from .models.decoded_model import Proxy, DecodedTransaction
 from .models.objects_model import Call
 from .providers import EtherscanProvider, Web3Provider
-from .providers.semantic_providers.semantics_database import (
-    MongoSemanticsDatabase,
+from .providers.semantic_providers import (
     ISemanticsDatabase,
+    SemanticsRepository,
+    MongoSemanticsDatabase,
 )
-from .providers.semantic_providers.semantics_repository import SemanticsRepository
 from .utils.validators import assert_tx_hash
 
 

ethtx/models/decoded_model.py

@@ -15,7 +15,7 @@
 from datetime import datetime
 from typing import List, Any, Optional
 
-from ethtx.models.objects_model import BlockMetadata, TransactionMetadata
+from ethtx.models.objects_model import BlockMetadata
 from ethtx.models.semantics_model import AddressSemantics, ERC20Semantics
 from ethtx.utils.pickable import JsonObject
 

ethtx/models/semantics_model.py

@@ -149,7 +149,7 @@ def __init__(
         name: str,
         events: Dict[str, EventSemantics],
         functions: Dict[str, FunctionSemantics],
-        transformations: [Dict[str, TransformationSemantics]],
+        transformations: [Dict[str, Dict[str, TransformationSemantics]]],
     ):
         self.code_hash = code_hash
         self.name = name

ethtx/providers/semantic_providers/__init__.py

@@ -0,0 +1,15 @@
+#  Copyright 2021 DAI Foundation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from .base import ISemanticsDatabase
+from .database import MongoSemanticsDatabase
+from .repository import SemanticsRepository

ethtx/providers/semantic_providers/base.py

@@ -0,0 +1,38 @@
+#  Copyright 2021 DAI Foundation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from abc import ABC
+from typing import Dict, Optional, Any, List
+
+
+class ISemanticsDatabase(ABC):
+    """Semantics Database. Represents raw interface required to be
+    implemented by a database that provides persistent
+    data about semantics"""
+
+    def get_address_semantics(self, chain_id: str, address: str) -> Optional[Dict]:
+        ...
+
+    def get_contract_semantics(self, code_hash: str) -> Optional[Dict]:
+        ...
+
+    def get_signature_semantics(self, signature_hash: str) -> Optional[List[Dict]]:
+        ...
+
+    def insert_contract(self, contract: dict, update_if_exist: bool = False) -> Any:
+        ...
+
+    def insert_address(self, address: dict, update_if_exist: bool = False) -> Any:
+        ...
+
+    def insert_signature(self, signature, update_if_exist: bool = False) -> Any:
+        ...

ethtx/providers/semantic_providers/const.py

@@ -0,0 +1,18 @@
+#  Copyright 2021 DAI Foundation
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from enum import Enum
+
+
+class MongoCollections(str, Enum):
+    ADDRESSES = "addresses"
+    CONTRACTS = "contracts"
+    SIGNATURES = "signatures"

ethtx/providers/semantic_providers/semantics_database.py → ethtx/providers/semantic_providers/database.py

@@ -10,54 +10,35 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from abc import ABC
-from typing import Dict, Optional, Any, List
+import logging
+from typing import Dict, Optional
 
 import bson
 from pymongo.cursor import Cursor
 from pymongo.database import Database as MongoDatabase
+from pymongo.errors import OperationFailure
 
+from .base import ISemanticsDatabase
+from .const import MongoCollections
 
-class ISemanticsDatabase(ABC):
-    """Semantics Database. Represents raw interface required to be
-    implemented by a database that provides persistent
-    data about semantics"""
+log = logging.getLogger(__name__)
 
-    def get_address_semantics(self, chain_id: str, address: str) -> Optional[Dict]:
-        ...
 
-    def get_contract_semantics(self, code_hash: str) -> Optional[Dict]:
-        ...
-
-    def get_signature_semantics(self, signature_hash: str) -> Optional[List[Dict]]:
-        ...
-
-    def insert_contract(self, contract: dict, update_if_exist: bool = False) -> Any:
-        ...
-
-    def insert_address(self, address: dict, update_if_exist: bool = False) -> Any:
-        ...
-
-    def insert_signature(self, signature, update_if_exist: bool = False) -> Any:
-        ...
+class MongoSemanticsDatabase(ISemanticsDatabase):
+    _db: MongoDatabase
 
+    def __init__(self, db: MongoDatabase):
+        self._db = db
 
-class MongoCollections:
-    ADDRESSES = "addresses"
-    CONTRACTS = "contracts"
-    SIGNATURES = "signatures"
+        self._addresses = None
+        self._contracts = None
+        self._signatures = None
 
+        self._init_collections()
 
-class MongoSemanticsDatabase(ISemanticsDatabase):
-    def get_collection_count(self):
+    def get_collection_count(self) -> int:
         return len(self._db.list_collection_names())
 
-    def __init__(self, db: MongoDatabase):
-        self._db = db
-        self._addresses = self._db["addresses"]
-        self._contracts = self._db["contracts"]
-        self._signatures = self._db["signatures"]
-
     def get_address_semantics(self, chain_id, address) -> Optional[Dict]:
         _id = f"{chain_id}-{address}"
         return self._addresses.find_one({"_id": _id}, {"_id": 0})
@@ -121,3 +102,17 @@ def insert_address(self, address, update_if_exist=False) -> Optional[bson.Object
 
         inserted_address = self._addresses.insert_one(address_with_id)
         return inserted_address.inserted_id
+
+    def _init_collections(self) -> None:
+        for mongo_collection in MongoCollections:
+            self.__setattr__(f"_{mongo_collection}", self._db[mongo_collection])
+
+            if mongo_collection == "signatures":
+                try:
+                    self._signatures.create_index(
+                        [("signature_hash", "TEXT"), ("name", "TEXT")],
+                        background=True,
+                        unique=True,
+                    )
+                except OperationFailure as e:
+                    log.warning(e)