diff --git a/.travis.yml b/.travis.yml index 178bd8bd..60feec7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,17 +26,10 @@ before_script: script: - coverage run --source=$SOURCE_FOLDER -m py.test -W ignore::DeprecationWarning deploy: - - provider: script - script: bash scripts/deploy_docker.sh staging - on: - branch: master - provider: script script: bash scripts/deploy_docker.sh develop on: - branch: develop - - provider: script - script: bash scripts/deploy_docker.sh $TRAVIS_TAG - on: - tags: true + branch: feature/issue-86 + after_success: - coveralls diff --git a/docker/web/run_web.sh b/docker/web/run_web.sh index 2f5b3862..a4b8e236 100755 --- a/docker/web/run_web.sh +++ b/docker/web/run_web.sh @@ -2,11 +2,14 @@ set -euo pipefail -echo "==> Migrating Django models ... " +echo "==> Migrating Django models..." python manage.py migrate --noinput +echo "==> Setup Gas Station..." python manage.py setup_gas_station +echo "==> Setup Safe Relay Task..." python manage.py setup_safe_relay if [ "${DEPLOY_MASTER_COPY_ON_INIT:-0}" = 1 ]; then + echo "==> Deploy Safe master copy..." python manage.py deploy_safe_master_copy fi diff --git a/requirements-test.txt b/requirements-test.txt index ff951b38..f78e9113 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,5 @@ -r requirements.txt -pytest==4.6.3 -pytest-django==3.5.0 +pytest==5.0.1 +pytest-django==3.5.1 pytest-sugar==0.9.2 coverage==4.5.3 diff --git a/requirements.txt b/requirements.txt index 4b763ca3..19312c8f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -Django==2.2.2 +Django==2.2.3 cachetools==3.1.1 celery==4.3.0 -django-authtools==1.6.0 +django-authtools==1.7.0 django-celery-beat==1.5.0 django-environ==0.4.5 django-filter==2.1.0 -django-model-utils==3.1.2 +django-model-utils==3.2.0 django-redis==4.10.0 djangorestframework-camel-case==1.0.3 djangorestframework==3.9.4 docutils==0.14 -drf-yasg[validation]==1.15.0 +drf-yasg[validation]==1.16.0 eth-abi==1.3.0 ethereum==2.3.2 factory-boy==2.12.0 @@ -19,11 +19,10 @@ gevent==1.4.0 gnosis-py==1.4.0 gunicorn==19.9.0 hexbytes==0.2.0 -jsonschema==2.6.0 lxml==4.2.5 numpy==1.16.4 packaging>=19.0 -psycopg2-binary==2.8.2 +psycopg2-binary==2.8.3 redis==3.2.1 requests==2.22.0 web3==4.9.2 diff --git a/safe_relay_service/gas_station/admin.py b/safe_relay_service/gas_station/admin.py index a5e8743d..3153d36a 100644 --- a/safe_relay_service/gas_station/admin.py +++ b/safe_relay_service/gas_station/admin.py @@ -7,4 +7,4 @@ class GasPriceAdmin(admin.ModelAdmin): date_hierarchy = 'created' list_display = ('created', 'lowest', 'safe_low', 'standard', 'fast', 'fastest') - ordering = ['created'] + ordering = ['-created'] diff --git a/safe_relay_service/relay/admin.py b/safe_relay_service/relay/admin.py index 71bd501b..8d9e9149 100644 --- a/safe_relay_service/relay/admin.py +++ b/safe_relay_service/relay/admin.py @@ -232,8 +232,10 @@ def safe_status(self, obj: SafeFunding): @admin.register(SafeMultisigTx) class SafeMultisigTxAdmin(admin.ModelAdmin): - list_display = ('safe_id', 'ethereum_tx_id', 'to', 'value', 'nonce', 'data') + date_hierarchy = 'created' + list_display = ('created', 'safe_id', 'ethereum_tx_id', 'to', 'value', 'nonce', 'data') list_filter = ('operation',) + ordering = ['-created'] search_fields = ['=safe__address', '=ethereum_tx__tx_hash', 'to'] diff --git a/safe_relay_service/relay/models.py b/safe_relay_service/relay/models.py index 5fc81dfe..0e555b43 100644 --- a/safe_relay_service/relay/models.py +++ b/safe_relay_service/relay/models.py @@ -1,13 +1,13 @@ import datetime -from datetime import timezone from enum import Enum from typing import Dict, List, Optional from django.contrib.postgres.fields import ArrayField, JSONField from django.db import models -from django.db.models import Case, DecimalField, F, Q, Sum, When -from django.db.models.expressions import OuterRef, RawSQL, Subquery -from django.db.models.functions import Coalesce +from django.db.models import (Avg, Case, Count, DurationField, F, Q, Sum, + Value, When) +from django.db.models.expressions import OuterRef, RawSQL, Subquery, Window +from django.db.models.functions import Cast, Coalesce, TruncDate from hexbytes import HexBytes from model_utils.models import TimeStampedModel @@ -17,6 +17,8 @@ Uint256Field) from gnosis.safe import SafeOperation +from .models_raw import SafeContractManagerRaw, SafeContractQuerySetRaw + class EthereumTxType(Enum): CALL = 0 @@ -52,27 +54,41 @@ def parse_call_type(call_type: str): return None -class SafeContractQuerySet(models.QuerySet): +class SafeContractManager(SafeContractManagerRaw): + def get_total_balance(self, from_date: datetime.datetime, to_date: datetime.datetime) -> int: + return int(self.with_balance().filter( + created__range=(from_date, to_date) + ).aggregate(total_balance=Sum('balance')).get('total_balance') or 0) + + +class SafeContractQuerySet(SafeContractQuerySetRaw): + deployed_filter = ~Q(safecreation2__block_number=None) | Q(safefunding__safe_deployed=True) + def with_balance(self): """ :return: Queryset with the Safes and a `balance` attribute """ - return self.annotate(balance=Subquery(InternalTx.objects.balance_for_all_safes( - ).filter(to=OuterRef('address')).values('balance').distinct(), DecimalField())) + return self.annotate( + balance=Subquery( + InternalTx.objects.balance_for_all_safes( + ).filter( + to=OuterRef('address') + ).values('balance').distinct(), + models.DecimalField())) def deployed(self): return self.filter( - ~Q(safecreation2__block_number=None) | Q(safefunding__safe_deployed=True) + self.deployed_filter ) def not_deployed(self): - return self.filter( - Q(safecreation2__block_number=None) & ~Q(safefunding__safe_deployed=True) + return self.exclude( + self.deployed_filter ) class SafeContract(TimeStampedModel): - objects = SafeContractQuerySet.as_manager() + objects = SafeContractManager.from_queryset(SafeContractQuerySet)() address = EthereumAddressField(primary_key=True) master_copy = EthereumAddressField() @@ -86,7 +102,22 @@ def get_tokens_with_balance(self) -> List[Dict[str, any]]: return EthereumEvent.objects.erc20_tokens_with_balance(self.address) +class SafeCreationManager(models.Manager): + def get_tokens_usage(self) -> Optional[List[Dict[str, any]]]: + """ + :return: List of Dict 'gas_token', 'total', 'number', 'percentage' + """ + total = self.deployed_and_checked().annotate(_x=Value(1)).values('_x').annotate(total=Count('_x') + ).values('total') + return self.deployed_and_checked().values('payment_token').annotate( + total=Subquery(total, output_field=models.IntegerField()) + ).annotate( + number=Count('safe_id'), percentage=Cast(100.0 * Count('pk') / F('total'), + models.FloatField())) + + class SafeCreation(TimeStampedModel): + objects = SafeCreationManager() deployer = EthereumAddressField(primary_key=True) safe = models.OneToOneField(SafeContract, on_delete=models.CASCADE) master_copy = EthereumAddressField() @@ -116,6 +147,20 @@ def wei_deploy_cost(self) -> int: class SafeCreation2Manager(models.Manager): + def get_tokens_usage(self) -> Optional[List[Dict[str, any]]]: + """ + :return: List of Dict 'gas_token', 'total', 'number', 'percentage' + """ + total = self.deployed_and_checked().annotate(_x=Value(1)).values('_x').annotate(total=Count('_x') + ).values('total') + return self.deployed_and_checked().values('payment_token').annotate( + total=Subquery(total, output_field=models.IntegerField()) + ).annotate( + number=Count('safe_id'), percentage=Cast(100.0 * Count('pk') / F('total'), + models.FloatField())) + + +class SafeCreation2QuerySet(models.QuerySet): def deployed_and_checked(self): return self.exclude( tx_hash=None, @@ -138,7 +183,7 @@ def pending_to_check(self): class SafeCreation2(TimeStampedModel): - objects = SafeCreation2Manager() + objects = SafeCreation2Manager.from_queryset(SafeCreation2QuerySet)() safe = models.OneToOneField(SafeContract, on_delete=models.CASCADE, primary_key=True) master_copy = EthereumAddressField() proxy_factory = EthereumAddressField() @@ -175,7 +220,7 @@ def wei_estimated_deploy_cost(self) -> int: return self.gas_estimated * self.gas_price_estimated -class SafeFundingManager(models.Manager): +class SafeFundingQuerySet(models.QuerySet): def pending_just_to_deploy(self): return self.filter( safe_deployed=False @@ -194,7 +239,7 @@ def not_deployed(self): class SafeFunding(TimeStampedModel): - objects = SafeFundingManager() + objects = SafeFundingQuerySet.as_manager() safe = models.OneToOneField(SafeContract, primary_key=True, on_delete=models.CASCADE) safe_funded = models.BooleanField(default=False) deployer_funded = models.BooleanField(default=False, db_index=True) # Set when deployer_funded_tx_hash is mined @@ -243,7 +288,7 @@ def create_from_block(self, block: Dict[str, any]) -> 'EthereumBlock': number=block['number'], gas_limit=block['gasLimit'], gas_used=block['gasUsed'], - timestamp=datetime.datetime.fromtimestamp(block['timestamp'], timezone.utc), + timestamp=datetime.datetime.fromtimestamp(block['timestamp'], datetime.timezone.utc), block_hash=block['hash'], ) @@ -297,9 +342,77 @@ def get_last_nonce_for_safe(self, safe_address: str) -> Optional[int]: nonce_dict = self.filter(safe=safe_address).order_by('-nonce').values('nonce').first() return nonce_dict['nonce'] if nonce_dict else None + def get_average_execution_time(self, from_date: datetime.datetime, + to_date: datetime.datetime) -> Optional[datetime.timedelta]: + return self.select_related( + 'ethereum_tx', 'ethereum_tx__block' + ).exclude( + ethereum_tx__block=None, + ).annotate( + interval=Cast(F('ethereum_tx__block__timestamp') - F('created'), + output_field=DurationField()) + ).filter( + created__range=(from_date, to_date) + ).aggregate(median=Avg('interval'))['median'] + + def get_average_execution_time_grouped(self, from_date: datetime.datetime, + to_date: datetime.datetime) -> Optional[datetime.timedelta]: + return self.select_related( + 'ethereum_tx', 'ethereum_tx__block' + ).exclude( + ethereum_tx__block=None, + ).annotate( + interval=Cast(F('ethereum_tx__block__timestamp') - F('created'), + output_field=DurationField()) + ).filter( + created__range=(from_date, to_date) + ).annotate( + created_date=TruncDate('created') + ).values( + 'created_date' + ).annotate( + median=Avg('interval') + ).values('created_date', 'median').order_by('created_date') + + def get_tokens_usage(self) -> Optional[List[Dict[str, any]]]: + """ + :return: List of Dict 'gas_token', 'total', 'number', 'percentage' + """ + total = self.annotate(_x=Value(1)).values('_x').annotate(total=Count('_x')).values('total') + return self.values( + 'gas_token' + ).annotate( + total=Subquery(total, output_field=models.IntegerField()) + ).annotate( + number=Count('pk'), percentage=Cast(100.0 * Count('pk') / F('total'), models.FloatField()) + ) + + def get_tokens_usage_grouped(self) -> Optional[List[Dict[str, any]]]: + """ + :return: List of Dict 'gas_token', 'total', 'number', 'percentage' + """ + return SafeMultisigTx.objects.annotate( + date=TruncDate('created') + ).annotate( + number=Window(expression=Count('*'), + partition_by=[F('gas_token'), F('date')]), + percentage=100.0 * Window(expression=Count('*'), + partition_by=[F('gas_token'), + F('date')] + ) / Window(expression=Count('*'), + partition_by=[F('date')]) + ).values( + 'date', 'gas_token', 'number', 'percentage' + ).distinct().order_by('date') + + +class SafeMultisigTxQuerySet(models.QuerySet): + def pending(self): + return self.filter(ethereum_tx__block=None) + class SafeMultisigTx(TimeStampedModel): - objects = SafeMultisigTxManager() + objects = SafeMultisigTxManager.from_queryset(SafeMultisigTxQuerySet)() safe = models.ForeignKey(SafeContract, on_delete=models.CASCADE, related_name='multisig_txs') ethereum_tx = models.ForeignKey(EthereumTx, on_delete=models.CASCADE, related_name='multisig_txs') to = EthereumAddressField(null=True, db_index=True) @@ -328,13 +441,13 @@ def balance_for_all_safes(self): # Exclude `DELEGATE_CALL` and errored transactions from `balance` calculations # There must be at least 2 txs (one in and another out for this method to work - # SELECT SUM(value) FROM "relay_internaltx" U0 WHERE U0."_from" = address; + # SELECT SUM(value) FROM "relay_internaltx" U0 WHERE U0."_from" = address # sum # ----- # (1 row) # But Django uses group by - # SELECT SUM(value) FROM "relay_internaltx" U0 WHERE U0."_from" = '0xE3726b0a9d59c3B28947Ae450e8B8FC864c7f77f' GROUP BY U0."_from"; + # SELECT SUM(value) FROM "relay_internaltx" U0 WHERE U0."_from" = '0xE3726b0a9d59c3B28947Ae450e8B8FC864c7f77f' GROUP BY U0."_from" # sum # ----- # (0 rows) @@ -361,8 +474,8 @@ def balance_for_all_safes(self): total=Coalesce(Sum('value'), 0) ).values('total') - return self.annotate(balance=Subquery(incoming_balance, output_field=DecimalField()) - - Subquery(outgoing_balance, output_field=DecimalField())) + return self.annotate(balance=Subquery(incoming_balance, output_field=models.DecimalField()) - + Subquery(outgoing_balance, output_field=models.DecimalField())) def calculate_balance(self, address: str) -> int: # balances_from = InternalTx.objects.filter(_from=safe_address).aggregate(value=Sum('value')).get('value', 0) @@ -421,7 +534,7 @@ def __str__(self): return 'Internal tx hash={} from={}'.format(self.ethereum_tx.tx_hash, self._from) -class SafeTxStatusManager(models.Manager): +class SafeTxStatusQuerySet(models.QuerySet): def deployed(self): return self.filter(safe__in=SafeContract.objects.deployed()) @@ -430,7 +543,7 @@ class SafeTxStatus(models.Model): """ Have information about the last scan for internal txs """ - objects = SafeTxStatusManager() + objects = SafeTxStatusQuerySet.as_manager() safe = models.OneToOneField(SafeContract, primary_key=True, on_delete=models.CASCADE) initial_block_number = models.IntegerField(default=0) # Block number when Safe creation process was started tx_block_number = models.IntegerField(default=0) # Block number when last internal tx scan ended @@ -467,12 +580,14 @@ def erc721_events(self, token_address: Optional[str] = None, address: Optional[s return self.erc20_and_721_events(token_address=token_address, address=address).filter(arguments__has_key='tokenId') + +class EthereumEventManager(models.Manager): def erc20_tokens_with_balance(self, address: str) -> List[Dict[str, any]]: """ :return: List of dictionaries {'token_address': str, 'balance': int} """ arguments_value_field = RawSQL("(arguments->>'value')::numeric", ()) - return EthereumEvent.objects.erc20_events( + return self.erc20_events( address=address ).values('token_address').annotate( balance=Sum(Case( @@ -516,7 +631,7 @@ def get_or_create_erc721_event(self, decoded_event: Dict[str, any]): class EthereumEvent(models.Model): - objects = EthereumEventQuerySet.as_manager() + objects = EthereumEventManager.from_queryset(EthereumEventQuerySet)() ethereum_tx = models.ForeignKey(EthereumTx, on_delete=models.CASCADE, related_name='events') log_index = models.PositiveIntegerField() token_address = EthereumAddressField(db_index=True) diff --git a/safe_relay_service/relay/models_raw.py b/safe_relay_service/relay/models_raw.py new file mode 100644 index 00000000..980eed51 --- /dev/null +++ b/safe_relay_service/relay/models_raw.py @@ -0,0 +1,246 @@ +import datetime +from decimal import Decimal +from typing import Dict, List, Optional + +from django.db import connection, models + +from gnosis.eth.constants import ERC20_721_TRANSFER_TOPIC + + +def parse_row(row): + """ + Remove Decimal from Raw SQL queries + """ + for r in row: + if isinstance(r, Decimal): + if r.as_integer_ratio()[1] == 1: + yield int(r) + else: + yield float(r) + else: + yield r + + +def run_raw_query(query: str, *arguments): + with connection.cursor() as cursor: + cursor.execute(query, arguments) + columns = [col[0] for col in cursor.description] + return [ + dict(zip(columns, parse_row(row))) + for row in cursor.fetchall() + ] + + +class SafeContractManagerRaw(models.Manager): + def get_average_deploy_time(self, from_date: datetime.datetime, to_date: datetime.datetime) -> datetime.timedelta: + query = """ + SELECT AVG(EB.timestamp - SC.created) + FROM (SELECT created, tx_hash FROM relay_safecreation + UNION SELECT created, tx_hash FROM relay_safecreation2) AS SC + JOIN relay_ethereumtx as ET ON SC.tx_hash=ET.tx_hash JOIN relay_ethereumblock as EB ON ET.block_id=EB.number + WHERE SC.created BETWEEN %s AND %s + """ + with connection.cursor() as cursor: + cursor.execute(query, [from_date, to_date]) + return cursor.fetchone()[0] + + def get_average_deploy_time_grouped(self, from_date: datetime.datetime, to_date: datetime.datetime) -> datetime.timedelta: + query = """ + SELECT DATE(SC.created) as created_date, AVG(EB.timestamp - SC.created) as average_deploy_time + FROM (SELECT created, tx_hash FROM relay_safecreation + UNION SELECT created, tx_hash FROM relay_safecreation2) AS SC + JOIN relay_ethereumtx as ET ON SC.tx_hash=ET.tx_hash JOIN relay_ethereumblock as EB ON ET.block_id=EB.number + WHERE SC.created BETWEEN %s AND %s + GROUP BY DATE(SC.created) + ORDER BY DATE(SC.created) + """ + + return run_raw_query(query, from_date, to_date) + + def get_total_balance_grouped(self, from_date: datetime.datetime, to_date: datetime.datetime) -> int: + """ + :return: Dictionary of {date: datetime.date, balance: decimal} + """ + query = """ + SELECT * FROM + (SELECT DISTINCT DATE(EB.timestamp) AS date, SUM(IT.value) OVER(ORDER BY DATE(EB.timestamp)) as balance + FROM (SELECT value, error, call_type, ethereum_tx_id + FROM relay_safecontract + JOIN relay_internaltx ON address="to" UNION + SELECT -value, error, call_type, ethereum_tx_id + FROM relay_safecontract + JOIN relay_internaltx ON address="_from") AS IT + JOIN relay_ethereumtx ET ON IT.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number + WHERE IT.error IS NULL AND IT.call_type != 1) AS RESULT + WHERE RESULT.date BETWEEN %s AND %s + ORDER BY RESULT.date + """ + return run_raw_query(query, from_date, to_date) + + def get_total_token_balance(self, from_date: datetime.datetime, to_date: datetime.datetime) -> Dict[str, any]: + """ + :return: Dictionary of {token_address: str, balance: decimal} + """ + query = """ + SELECT token_address, SUM(EE.value) as balance FROM + (SELECT SC.created, ethereum_tx_id, address, token_address, -(arguments->>'value')::decimal AS value + FROM relay_safecontract SC JOIN relay_ethereumevent EV + ON SC.address = EV.arguments->>'from' + WHERE arguments ? 'value' AND topic='{0}' + UNION SELECT SC.created, ethereum_tx_id, address, token_address, (arguments->>'value')::decimal + FROM relay_safecontract SC JOIN relay_ethereumevent EV + ON SC.address = EV.arguments->>'to' + WHERE arguments ? 'value' AND topic='{0}') AS EE + WHERE EE.created BETWEEN %s AND %s + GROUP BY token_address + """.format(ERC20_721_TRANSFER_TOPIC.replace('0x', '')) # No risk of SQL Injection + + return run_raw_query(query, from_date, to_date) + + def get_total_token_balance_grouped(self, from_date: datetime.datetime, to_date: datetime.datetime) -> Dict[str, any]: + """ + :return: Dictionary of {date: datetime.date, token_address: str, balance: decimal} + """ + query = """ + SELECT * + FROM (SELECT DISTINCT + DATE(EB.timestamp) as date, + token_address, + SUM(EE.value) OVER(PARTITION BY token_address ORDER BY DATE(EB.timestamp)) as balance + FROM (SELECT SC.created, ethereum_tx_id, address, token_address, -(arguments->>'value')::decimal AS value + FROM relay_safecontract SC JOIN relay_ethereumevent EV + ON SC.address = EV.arguments->>'from' + WHERE arguments ? 'value' AND topic='{0}' + UNION SELECT SC.created, ethereum_tx_id, address, token_address, (arguments->>'value')::decimal + FROM relay_safecontract SC JOIN relay_ethereumevent EV + ON SC.address = EV.arguments->>'to' + WHERE arguments ? 'value' AND topic='{0}') AS EE + JOIN relay_ethereumtx ET ON EE.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number) AS RESULT + WHERE RESULT.date BETWEEN %s AND %s + ORDER BY RESULT.date; + """.format(ERC20_721_TRANSFER_TOPIC.replace('0x', '')) # No risk of SQL Injection + + return run_raw_query(query, from_date, to_date) + + def get_total_volume(self, from_date: datetime.datetime, to_date: datetime.datetime) -> int: + from .models import EthereumTxCallType + query = """ + SELECT SUM(IT.value) AS value + FROM relay_safecontract SC + JOIN relay_internaltx IT ON SC.address=IT."_from" OR SC.address=IT."to" + JOIN relay_ethereumtx ET ON IT.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number + WHERE IT.call_type != {0} + AND error IS NULL + AND EB.timestamp BETWEEN %s AND %s + """.format(EthereumTxCallType.DELEGATE_CALL.value) + with connection.cursor() as cursor: + cursor.execute(query, [from_date, to_date]) + value = cursor.fetchone()[0] + if value is not None: + return int(value) + + def get_total_volume_grouped(self, from_date: datetime.datetime, to_date: datetime.datetime) -> int: + from .models import EthereumTxCallType + query = """ + SELECT DATE(EB.timestamp) as date, + SUM(IT.value) AS value + FROM relay_safecontract SC + JOIN relay_internaltx IT ON SC.address=IT."_from" OR SC.address=IT."to" + JOIN relay_ethereumtx ET ON IT.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number + WHERE IT.call_type != {0} + AND error IS NULL + AND EB.timestamp BETWEEN %s AND %s + GROUP BY DATE(EB.timestamp) + ORDER BY DATE(EB.timestamp) + """.format(EthereumTxCallType.DELEGATE_CALL.value) + + return run_raw_query(query, from_date, to_date) + + def get_total_token_volume(self, from_date: datetime.datetime, to_date: datetime.datetime): + """ + :return: Dictionary of {token_address: str, volume: int} + """ + query = """ + SELECT EV.token_address, SUM((EV.arguments->>'value')::decimal) AS value + FROM relay_safecontract SC + JOIN relay_ethereumevent EV ON SC.address = EV.arguments->>'from' OR SC.address = EV.arguments->>'to' + JOIN relay_ethereumtx ET ON EV.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number + WHERE arguments ? 'value' + AND topic='{0}' + AND EB.timestamp BETWEEN %s AND %s + GROUP BY token_address""".format(ERC20_721_TRANSFER_TOPIC.replace('0x', '')) # No risk of SQL Injection + + return run_raw_query(query, from_date, to_date) + + def get_total_token_volume_grouped(self, from_date: datetime.datetime, to_date: datetime.datetime): + """ + :return: Dictionary of {token_address: str, volume: int} + """ + query = """ + SELECT DATE(EB.timestamp) as date, EV.token_address, SUM((EV.arguments->>'value')::decimal) AS value + FROM relay_safecontract SC + JOIN relay_ethereumevent EV ON SC.address = EV.arguments->>'from' OR SC.address = EV.arguments->>'to' + JOIN relay_ethereumtx ET ON EV.ethereum_tx_id=ET.tx_hash + JOIN relay_ethereumblock EB ON ET.block_id=EB.number + WHERE arguments ? 'value' + AND topic='{0}' + AND EB.timestamp BETWEEN %s AND %s + GROUP BY DATE(EB.timestamp), token_address + ORDER BY DATE(EB.timestamp)""".format(ERC20_721_TRANSFER_TOPIC.replace('0x', '')) # No risk of SQL Injection + + return run_raw_query(query, from_date, to_date) + + def get_creation_tokens_usage(self, from_date: datetime.datetime, + to_date: datetime.datetime) -> Optional[List[Dict[str, any]]]: + query = """ + SELECT DISTINCT payment_token, COUNT(*) OVER(PARTITION BY payment_token) as number, + 100.0 * COUNT(*) OVER(PARTITION BY payment_token) / COUNT(*) OVER() as percentage + FROM (SELECT tx_hash, payment_token, created FROM relay_safecreation + UNION SELECT tx_hash, payment_token, created FROM relay_safecreation2) SC + JOIN relay_ethereumtx ET ON SC.tx_hash = ET.tx_hash + WHERE SC.created BETWEEN %s AND %s + """ + + return run_raw_query(query, from_date, to_date) + + def get_creation_tokens_usage_grouped(self, from_date: datetime.datetime, + to_date: datetime.datetime) -> Optional[List[Dict[str, any]]]: + query = """ + SELECT DISTINCT DATE(SC.created), payment_token, + COUNT(*) OVER(PARTITION BY DATE(SC.created), payment_token) as number, + 100.0 * COUNT(*) OVER(PARTITION BY DATE(SC.created), payment_token) / + COUNT(*) OVER(PARTITION BY DATE(SC.created)) as percentage + FROM (SELECT tx_hash, payment_token, created FROM relay_safecreation + UNION SELECT tx_hash, payment_token, created FROM relay_safecreation2) SC + JOIN relay_ethereumtx ET ON SC.tx_hash = ET.tx_hash + WHERE SC.created BETWEEN %s AND %s + ORDER BY(DATE(SC.created)) + """ + # Returns list of {'date': date, 'payment_token': Optional[str], 'number': int, percentage: 'float') + return run_raw_query(query, from_date, to_date) + + +class SafeContractQuerySetRaw(models.QuerySet): + def with_token_balance(self): + """ + :return: Dictionary of {address: str, token_address: str and balance: int} + """ + query = """ + SELECT address, token_address, SUM(value) as balance FROM + (SELECT address, token_address, -(arguments->>'value')::decimal AS value + FROM relay_safecontract JOIN relay_ethereumevent + ON relay_safecontract.address = relay_ethereumevent.arguments->>'from' + WHERE arguments ? 'value' AND topic='{0}' + UNION SELECT address, token_address, (arguments->>'value')::decimal + FROM relay_safecontract JOIN relay_ethereumevent + ON relay_safecontract.address = relay_ethereumevent.arguments->>'to' + WHERE arguments ? 'value' AND topic='{0}') AS X + GROUP BY address, token_address + """.format(ERC20_721_TRANSFER_TOPIC.replace('0x', '')) + + return run_raw_query(query) diff --git a/safe_relay_service/relay/services/__init__.py b/safe_relay_service/relay/services/__init__.py index 0aebc220..cdda5089 100644 --- a/safe_relay_service/relay/services/__init__.py +++ b/safe_relay_service/relay/services/__init__.py @@ -6,4 +6,5 @@ NotificationServiceProvider) from .safe_creation_service import (SafeCreationService, SafeCreationServiceProvider) +from .stats_service import StatsService, StatsServiceProvider from .transaction_service import TransactionService, TransactionServiceProvider diff --git a/safe_relay_service/relay/services/stats_service.py b/safe_relay_service/relay/services/stats_service.py new file mode 100644 index 00000000..fd689ba9 --- /dev/null +++ b/safe_relay_service/relay/services/stats_service.py @@ -0,0 +1,107 @@ +import datetime +from logging import getLogger +from typing import Dict + +from django.db.models import Count +from django.db.models.functions import TruncDate +from django.utils import timezone + +from pytz import utc + +from gnosis.eth import EthereumClient, EthereumClientProvider + +from safe_relay_service.gas_station.gas_station import (GasStation, + GasStationProvider) + +from ..models import SafeContract, SafeCreation2, SafeMultisigTx + +logger = getLogger(__name__) + + +class StatsServiceProvider: + def __new__(cls): + if not hasattr(cls, 'instance'): + cls.instance = StatsService(EthereumClientProvider(), GasStationProvider()) + return cls.instance + + @classmethod + def del_singleton(cls): + if hasattr(cls, "instance"): + del cls.instance + + +class StatsService: + def __init__(self, ethereum_client: EthereumClient, gas_station: GasStation): + self.ethereum_client = ethereum_client + self.gas_station = gas_station + + def get_gas_price_stats(self) -> Dict[str, any]: + pass + + def get_relay_history_stats(self, from_date: datetime.datetime = None, + to_date: datetime.datetime = None) -> Dict[str, any]: + + from_date = from_date if from_date else datetime.datetime(2018, 1, 1, tzinfo=utc) + to_date = to_date if to_date else timezone.now() + + def add_time_filter(queryset): + return queryset.filter(created__range=(from_date, to_date)) + + return { + 'safes_created': { + 'deployed': add_time_filter(SafeContract.objects.deployed()).annotate( + created_date=TruncDate('created')).values('created_date').annotate(number=Count('*') + ).order_by('created'), + 'average_deploy_time_seconds': SafeContract.objects.get_average_deploy_time_grouped(from_date, to_date), + 'payment_tokens': SafeContract.objects.get_creation_tokens_usage_grouped(from_date, to_date), + 'funds_stored': { + 'ether': SafeContract.objects.get_total_balance_grouped(from_date, to_date), + 'tokens': SafeContract.objects.get_total_token_balance_grouped(from_date, to_date), + } + }, + 'relayed_txs': { + 'total': add_time_filter(SafeMultisigTx.objects.annotate( + created_date=TruncDate('created')).values('created_date').annotate(number=Count('*') + ).order_by('created')), + 'average_execution_time_seconds': SafeMultisigTx.objects.get_average_execution_time_grouped(from_date, + to_date), + 'payment_tokens': add_time_filter(SafeMultisigTx.objects.get_tokens_usage_grouped()), + 'volume': { + 'ether': SafeContract.objects.get_total_volume_grouped(from_date, to_date), + 'tokens': SafeContract.objects.get_total_token_volume_grouped(from_date, to_date), + } + } + } + + def get_relay_stats(self, from_date: datetime.datetime = None, + to_date: datetime.datetime = None) -> Dict[str, any]: + + from_date = from_date if from_date else datetime.datetime(2018, 1, 1, tzinfo=utc) + to_date = to_date if to_date else timezone.now() + + def add_time_filter(queryset): + return queryset.filter(created__range=(from_date, to_date)) + + deployed = add_time_filter(SafeContract.objects.deployed()).count() + return { + 'safes_created': { + 'deployed': deployed, + 'not_deployed': add_time_filter(SafeContract.objects.all()).count() - deployed, + 'average_deploy_time_seconds': SafeContract.objects.get_average_deploy_time(from_date, to_date), + 'payment_tokens': SafeContract.objects.get_creation_tokens_usage(from_date, to_date), + 'funds_stored': { + 'ether': SafeContract.objects.get_total_balance(from_date, to_date), #FIXME + 'tokens': SafeContract.objects.get_total_token_balance(from_date, to_date), #FIXME + } + }, + 'relayed_txs': { + 'total': add_time_filter(SafeMultisigTx.objects.all()).count(), + 'average_execution_time_seconds': SafeMultisigTx.objects.get_average_execution_time(from_date, to_date), + 'pending_txs': add_time_filter(SafeMultisigTx.objects.pending()).count(), + 'payment_tokens': add_time_filter(SafeMultisigTx.objects.get_tokens_usage()), + 'volume': { + 'ether': SafeContract.objects.get_total_volume(from_date, to_date), + 'tokens': SafeContract.objects.get_total_token_volume(from_date, to_date), + } + } + } diff --git a/safe_relay_service/relay/tests/test_models.py b/safe_relay_service/relay/tests/test_models.py index 35fae786..16e8e6a4 100644 --- a/safe_relay_service/relay/tests/test_models.py +++ b/safe_relay_service/relay/tests/test_models.py @@ -1,18 +1,23 @@ +import datetime + from django.test import TestCase +from django.utils import timezone from eth_account import Account from hexbytes import HexBytes +from pytz import utc from web3 import Web3 from gnosis.eth.constants import NULL_ADDRESS from ..models import (EthereumEvent, EthereumTxCallType, InternalTx, - SafeContract, SafeFunding) -from .factories import (EthereumEventFactory, InternalTxFactory, - SafeCreation2Factory, SafeFundingFactory) + SafeContract, SafeFunding, SafeMultisigTx) +from .factories import (EthereumEventFactory, EthereumTxFactory, + InternalTxFactory, SafeCreation2Factory, + SafeFundingFactory, SafeMultisigTxFactory) -class TestModels(TestCase): +class TestSafeContractModel(TestCase): def test_hex_field(self): safe_address = Account.create().address safe = SafeContract.objects.create(address=safe_address) @@ -58,7 +63,22 @@ def test_safe_contract_deployed(self): self.assertEqual(SafeContract.objects.deployed().count(), 2) self.assertIn(safe_creation_2.safe.address, [s.address for s in SafeContract.objects.deployed()]) - def test_ethereum_event_model(self): + def test_get_average_deploy_time(self): + from_date = datetime.datetime(2018, 1, 1, tzinfo=utc) + to_date = timezone.now() + self.assertIsNone(SafeContract.objects.get_average_deploy_time(from_date, to_date)) + ethereum_tx = EthereumTxFactory() + self.assertIsNone(SafeContract.objects.get_average_deploy_time(from_date, to_date)) + interval = datetime.timedelta(seconds=10) + safe_creation = SafeCreation2Factory(created=ethereum_tx.block.timestamp - interval, + tx_hash=ethereum_tx.tx_hash) + from_date = safe_creation.created - interval + to_date = safe_creation.created + interval + self.assertEqual(SafeContract.objects.get_average_deploy_time(from_date, to_date), interval) + + +class TestEthereumEventModel(TestCase): + def test_ethereum_event(self): self.assertEqual(EthereumEvent.objects.count(), 0) # Create ERC20 Event @@ -74,6 +94,8 @@ def test_ethereum_event_model(self): self.assertTrue(EthereumEvent.objects.erc20_events().get().is_erc20()) self.assertTrue(EthereumEvent.objects.erc721_events().get().is_erc721()) + +class TestInternalTxModel(TestCase): def test_internal_tx_balance(self): address = Account.create().address value = Web3.toWei(1, 'ether') @@ -100,3 +122,22 @@ def test_internal_tx_balance(self): # More income InternalTxFactory(to=address, value=1) self.assertEqual(InternalTx.objects.calculate_balance(address), 2) + + +class TestSafeMultisigTxModel(TestCase): + def test_get_average_execution_time(self): + from_date = datetime.datetime(2018, 1, 1, tzinfo=utc) + to_date = timezone.now() + self.assertIsNone(SafeMultisigTx.objects.get_average_execution_time(from_date, to_date)) + safe_multisig_tx = SafeMultisigTxFactory() + interval = datetime.timedelta(seconds=10) + safe_multisig_tx.ethereum_tx.block.timestamp = safe_multisig_tx.created + interval + safe_multisig_tx.ethereum_tx.block.save() + from_date = safe_multisig_tx.created - interval + to_date = safe_multisig_tx.created + interval + self.assertEqual(SafeMultisigTx.objects.get_average_execution_time(from_date, to_date), interval) + safe_multisig_tx_2 = SafeMultisigTxFactory() + interval_2 = datetime.timedelta(seconds=5) + safe_multisig_tx_2.ethereum_tx.block.timestamp = safe_multisig_tx_2.created + interval_2 + safe_multisig_tx_2.ethereum_tx.block.save() + self.assertEqual(SafeMultisigTx.objects.get_average_execution_time(from_date, to_date), (interval + interval_2) / 2) diff --git a/safe_relay_service/relay/tests/test_stats_service.py b/safe_relay_service/relay/tests/test_stats_service.py new file mode 100644 index 00000000..2e4ebbda --- /dev/null +++ b/safe_relay_service/relay/tests/test_stats_service.py @@ -0,0 +1,14 @@ +from django.test import TestCase + +from ..services import StatsServiceProvider +from .relay_test_case import RelayTestCaseMixin + + +class TestStatsService(RelayTestCaseMixin, TestCase): + def test_get_relay_history_stats(self): + stats_service = StatsServiceProvider() + self.assertIsNotNone(stats_service.get_relay_history_stats()) + + def test_get_relay_stats(self): + stats_service = StatsServiceProvider() + self.assertIsNotNone(stats_service.get_relay_stats()) diff --git a/safe_relay_service/relay/urls.py b/safe_relay_service/relay/urls.py index 6fbf8cc1..4700879e 100644 --- a/safe_relay_service/relay/urls.py +++ b/safe_relay_service/relay/urls.py @@ -29,6 +29,8 @@ name='safe-multisig-tx-estimate'), path('safes//transactions/estimates/', views.SafeMultisigTxEstimatesView.as_view(), name='safe-multisig-tx-estimates'), + path('stats/', views.StatsView.as_view(), name='stats'), + path('stats/history/', views.StatsHistoryView.as_view(), name='stats-history'), path('private/api-token-auth/', rest_views.obtain_auth_token, name='api-token-auth'), path('private/safes/', views.PrivateSafesView.as_view(), name='private-safes'), ] diff --git a/safe_relay_service/relay/views.py b/safe_relay_service/relay/views.py index 955af085..f02cd447 100644 --- a/safe_relay_service/relay/views.py +++ b/safe_relay_service/relay/views.py @@ -2,8 +2,10 @@ from django.conf import settings from django.db.models import Q +from django.utils.dateparse import parse_datetime from django_filters.rest_framework import DjangoFilterBackend +from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema from eth_account.account import Account from rest_framework import filters, status @@ -33,6 +35,7 @@ SafeMultisigTxResponseSerializer, SafeRelayMultisigTxSerializer, SafeResponseSerializer, TransactionEstimationWithNonceAndGasTokensResponseSerializer) +from .services import StatsServiceProvider from .services.funding_service import FundingServiceException from .services.safe_creation_service import (SafeCreationServiceException, SafeCreationServiceProvider) @@ -435,6 +438,49 @@ def get_queryset(self): ).select_related('ethereum_tx', 'ethereum_tx__block') +class StatsView(APIView): + permission_classes = (AllowAny,) + # serializer_class = StatsResponseSerializer + + @swagger_auto_schema(manual_parameters=[ + openapi.Parameter('fromDate', openapi.IN_QUERY, type=openapi.TYPE_STRING, format='date-time', + description="ISO 8601 date to filter stats from. If not set, 2018-01-01"), + openapi.Parameter('toDate', openapi.IN_QUERY, type=openapi.TYPE_STRING, format='date-time', + description="ISO 8601 date to filter stats to. If not set, now"), + ]) + def get(self, request, format=None): + """ + Get stats of the Safe Relay Service + """ + from_date = self.request.query_params.get('fromDate') + to_date = self.request.query_params.get('toDate') + from_date = parse_datetime(from_date) if from_date else from_date + to_date = parse_datetime(to_date) if to_date else to_date + return Response(status=status.HTTP_200_OK, data=StatsServiceProvider().get_relay_stats(from_date, to_date)) + + +class StatsHistoryView(APIView): + permission_classes = (AllowAny,) + # serializer_class = StatsResponseSerializer + + @swagger_auto_schema(manual_parameters=[ + openapi.Parameter('fromDate', openapi.IN_QUERY, type=openapi.TYPE_STRING, format='date-time', + description="ISO 8601 date to filter stats from. If not set, 2018-01-01"), + openapi.Parameter('toDate', openapi.IN_QUERY, type=openapi.TYPE_STRING, format='date-time', + description="ISO 8601 date to filter stats to. If not set, now"), + ]) + def get(self, request, format=None): + """ + Get historic stats of the Safe Relay Service + """ + from_date = self.request.query_params.get('fromDate') + to_date = self.request.query_params.get('toDate') + from_date = parse_datetime(from_date) if from_date else from_date + to_date = parse_datetime(to_date) if to_date else to_date + return Response(status=status.HTTP_200_OK, + data=StatsServiceProvider().get_relay_history_stats(from_date, to_date)) + + class PrivateSafesView(ListAPIView): authentication_classes = (TokenAuthentication,) permission_classes = (IsAuthenticated,) diff --git a/safe_relay_service/schema_validator/__init__.py b/safe_relay_service/schema_validator/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/safe_relay_service/schema_validator/tests/__init__.py b/safe_relay_service/schema_validator/tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/safe_relay_service/schema_validator/tests/data/test_schema.json b/safe_relay_service/schema_validator/tests/data/test_schema.json deleted file mode 100644 index 4da2fc1d..00000000 --- a/safe_relay_service/schema_validator/tests/data/test_schema.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "params": { - "type": "object", - "properties": { - "safe": { - "type": "string" - }, - "deployer": { - "type": "string" - } - }, - "required": ["safe", "deployer"] - } - }, - "required": ["type", "params"] -} \ No newline at end of file diff --git a/safe_relay_service/schema_validator/tests/test_validator.py b/safe_relay_service/schema_validator/tests/test_validator.py deleted file mode 100644 index adf12284..00000000 --- a/safe_relay_service/schema_validator/tests/test_validator.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -from django.test import TestCase - -from jsonschema.exceptions import ValidationError - -from safe_relay_service.schema_validator.validator import Validator - - -class TestValidator(TestCase): - def setUp(self): - self.dirpath = os.path.dirname(__file__) + '/data' - - def test_validator_works(self): - valid_data = { - "type": "safeCreation", - "params": { - "safe": "0x0", - "deployer": "0x0" - } - } - - invalid_data = { - "params": { - "safe": "0x0", - "deployer": "0x0" - } - } - validator = Validator(base_path=self.dirpath) - validator.load_schema('test_schema.json') - - # Checks - # .validate() returns nothing or raises a ValidationError - is_valid = validator.validate(valid_data) - self.assertIsNone(is_valid) - with self.assertRaises(ValidationError): - validator.validate(invalid_data) - - def test_schema_not_found(self): - validator = Validator(base_path=self.dirpath + '/') - with self.assertRaises(FileNotFoundError): - validator.load_schema('not_existing_schema.json') diff --git a/safe_relay_service/schema_validator/validator.py b/safe_relay_service/schema_validator/validator.py deleted file mode 100644 index 4c332535..00000000 --- a/safe_relay_service/schema_validator/validator.py +++ /dev/null @@ -1,88 +0,0 @@ -import datetime -import json - -from jsonschema import Draft4Validator, validate, validators -from jsonschema.exceptions import ValidationError - -from safe_relay_service.utils.singleton import singleton - - -@singleton -class Validator(object): - """JSON Schema Validator class""" - - def __init__(self, base_path='schemas'): - if base_path[-1] != '/': - base_path = base_path + '/' - - self._base_path = base_path - self._schema = None - self._custom_validator = None - - def load_schema(self, file_name): - """Loads a JSON Schema - Args: - file_name: the JSON file name, along with its .json exstension - Raises: - IOError: if the file doesn't exists - ValueError: if the json file isn't well formatted - """ - if file_name[0] == '/': - file_name = file_name[1:] - - with open(self._base_path + file_name) as f: - self._schema = json.load(f) - - def extend_validator(self, name): - """Sets a custom validator extending a Draft4Validator - Args: - name: the validator name - Raises: - Exception: if the validator doesn't exists for the given name - """ - custom_validator = self.get_custom_validator(name) - - if not custom_validator: - raise Exception('%s validator is not available' % name) - else: - new_validators = {name: custom_validator} - self._custom_validator = validators.extend(Draft4Validator, new_validators) - - def get_custom_validator(self, name): - """Returns a suitable jsonschema custom validator function - Args: - name: the validator name - Returns: - The custom validator function, None otherwise. - """ - if name == 'date-time': - def date_time_validator(validator, format, instance, schema): - if not validator.is_type(instance, "string"): - return - try: - datetime.datetime.strptime(instance, format) - except ValueError as ve: - yield ValidationError(ve.message) - - return date_time_validator - - return None - - def validate(self, data): - """Validates a dictionary against a schema - Args: - data: A dictionary - Returns: - Nothing for success, Exception otherwise. - Raises: - Exception: if schema is not provided - jsonschema.exceptions.ValidationError: if data is cannot be validated - """ - if not self._schema: - raise Exception('Schema dictionary not provided') - elif self._custom_validator: - # Validate returns nothing - self._custom_validator(self._schema).validate(data) - else: - # Validate returns nothing - validate(data, self._schema) diff --git a/safe_relay_service/tokens/models.py b/safe_relay_service/tokens/models.py index 63b7b823..037523da 100644 --- a/safe_relay_service/tokens/models.py +++ b/safe_relay_service/tokens/models.py @@ -42,13 +42,13 @@ def _price(self) -> Optional[float]: price = property(_price) -class TokenManager(models.Manager): +class TokenQuerySet(models.QuerySet): def gas_tokens(self): return self.filter(gas=True) class Token(models.Model): - objects = TokenManager() + objects = TokenQuerySet.as_manager() address = EthereumAddressField(primary_key=True) name = models.CharField(max_length=30) symbol = models.CharField(max_length=30)