Dex Aggregator Migration - #1 - Lifi (duneanalytics#4054)

* Dex Aggregator Migration - #1 - Lifi * push dex aggregator seed * revert changes to legacy file * test fix * test * test * smol fix * TINY FIXES AND CLEAN UP * clean --------- Co-authored-by: Huang Geyang <Sukebeta@outlook.com>
Steakhouse-Financial · Aug 21, 2023 · ab34874 · ab34874
1 parent f0f5124
commit ab34874
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 30 deletions.
diff --git a/models/lifi/fantom/lifi_fantom_schema.yml b/models/lifi/fantom/lifi_fantom_schema.yml
@@ -21,7 +21,7 @@ models:
             - tx_hash
             - evt_index
             - trace_address
-      - check_dex_aggregator_seed_legacy:
+      - check_dex_aggregator_seed:
           blockchain: fantom
           project: lifi
           version: 2
@@ -95,6 +95,9 @@ models:
       - &evt_index
         name: evt_index
         description: ""
+      - &block_month
+        name: block_month
+        description: "UTC event block month of each DEX trade"
 
   - name: lifi_fantom_trades
     meta:
@@ -130,3 +133,4 @@ models:
       - *tx_to
       - *trace_address
       - *evt_index
+      - *block_month
diff --git a/models/lifi/fantom/lifi_fantom_trades.sql b/models/lifi/fantom/lifi_fantom_trades.sql
@@ -1,5 +1,6 @@
 
 {{ config(
+    tags=['dunesql'],
         alias = alias('trades')
         )
 }}
@@ -17,6 +18,7 @@ FROM (
         project,
         version,
         block_date,
+        block_month,
         block_time,
         token_bought_symbol,
         token_sold_symbol,
@@ -42,4 +44,3 @@ FROM (
     {% endif %}
     {% endfor %}
 )
-;
diff --git a/models/lifi/fantom/lifi_fantom_trades_legacy.sql b/models/lifi/fantom/lifi_fantom_trades_legacy.sql
@@ -44,4 +44,4 @@ FROM (
     {% endif %}
     {% endfor %}
 )
-;
+;
diff --git a/models/lifi/fantom/lifi_v2_fantom_trades.sql b/models/lifi/fantom/lifi_v2_fantom_trades.sql
@@ -1,7 +1,8 @@
 {{ config(
+    tags=['dunesql'],
     schema = 'lifi_v2_fantom',
     alias = alias('trades'),
-    partition_by = ['block_date'],
+    partition_by = ['block_month'],
     materialized = 'incremental',
     file_format = 'delta',
     incremental_strategy = 'merge',
@@ -26,27 +27,27 @@ dexs as (
     {% for trade_tables in trade_event_tables %}
         SELECT 
             evt_block_time as block_time,
-            '' as maker, 
+            CAST(NULL as VARBINARY) as maker, 
             toAmount as token_bought_amount_raw,
             fromAmount as token_sold_amount_raw,
             CAST(NULL as double) as amount_usd,
             CASE 
-                WHEN toAssetId IN ('0', 'O', '0x0000000000000000000000000000000000000000')
-                THEN '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' -- wftm 
+                WHEN toAssetId IN (0x, 0x0000000000000000000000000000000000000000)
+                THEN 0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83 -- wftm 
                 ELSE toAssetId
             END as token_bought_address,
             CASE 
-                WHEN fromAssetId IN ('0', 'O', '0x0000000000000000000000000000000000000000')
-                THEN '0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83' -- wftm 
+                WHEN fromAssetId IN (0x, 0x0000000000000000000000000000000000000000)
+                THEN 0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83 -- wftm 
                 ELSE fromAssetId
             END as token_sold_address,
             contract_address as project_contract_address,
             evt_tx_hash as tx_hash, 
-            CAST(ARRAY() as array<bigint>) AS trace_address,
+            ARRAY[-1] AS trace_address,
             evt_index
         FROM {{ trade_tables }} p 
         {% if is_incremental() %}
-        WHERE p.evt_block_time >= date_trunc("day", now() - interval '1 week')
+        WHERE p.evt_block_time >= date_trunc('day', now() - interval '7' Day)
         {% endif %}
         {% if not loop.last %}
         UNION ALL
@@ -57,7 +58,8 @@ SELECT
     'fantom' as blockchain,
     'lifi' as project,
     '2' as version,
-    try_cast(date_trunc('DAY', dexs.block_time) as date) as block_date,
+    cast(date_trunc('DAY', dexs.block_time) as date) as block_date,
+    cast(date_trunc('MONTH', dexs.block_time) as date) as block_month,
     dexs.block_time,
     erc20a.symbol as token_bought_symbol,
     erc20b.symbol as token_sold_symbol,
@@ -67,31 +69,31 @@ SELECT
     end as token_pair,
     dexs.token_bought_amount_raw / power(10, erc20a.decimals) AS token_bought_amount,
     dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount,
-    CAST(dexs.token_bought_amount_raw AS DECIMAL(38,0)) AS token_bought_amount_raw,
-    CAST(dexs.token_sold_amount_raw AS DECIMAL(38,0)) AS token_sold_amount_raw,
+    dexs.token_bought_amount_raw,
+    dexs.token_sold_amount_raw,
     coalesce(
         dexs.amount_usd
         ,(dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price
         ,(dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price
     ) AS amount_usd,
     dexs.token_bought_address,
     dexs.token_sold_address,
-    tx.from AS taker, -- no taker in swap event
+    tx."from" AS taker, -- no taker in swap event
     dexs.maker,
     dexs.project_contract_address,
     dexs.tx_hash,
-    tx.from AS tx_from,
+    tx."from" AS tx_from,
     tx.to AS tx_to,
     dexs.trace_address,
     dexs.evt_index
 from dexs
 inner join {{ source('fantom', 'transactions') }} tx
     on dexs.tx_hash = tx.hash
     {% if not is_incremental() %}
-    and tx.block_time >= '{{project_start_date}}'
+    and tx.block_time >= TIMESTAMP '{{project_start_date}}'
     {% endif %}
     {% if is_incremental() %}
-    and tx.block_time >= date_trunc("day", now() - interval '1 week')
+    and tx.block_time >= date_trunc('day', now() - interval '7' Day)
     {% endif %}
 left join {{ ref('tokens_erc20') }} erc20a
     on erc20a.contract_address = dexs.token_bought_address
@@ -104,18 +106,18 @@ left join {{ source('prices', 'usd') }} p_bought
     and p_bought.contract_address = dexs.token_bought_address
     and p_bought.blockchain = 'fantom'
     {% if not is_incremental() %}
-    and p_bought.minute >= '{{project_start_date}}'
+    and p_bought.minute >= TIMESTAMP '{{project_start_date}}'
     {% endif %}
     {% if is_incremental() %}
-    and p_bought.minute >= date_trunc("day", now() - interval '1 week')
+    and p_bought.minute >= date_trunc('day', now() - interval '7' Day)
     {% endif %}
 left join {{ source('prices', 'usd') }} p_sold
     on p_sold.minute = date_trunc('minute', dexs.block_time)
     and p_sold.contract_address = dexs.token_sold_address
     and p_sold.blockchain = 'fantom'
     {% if not is_incremental() %}
-    and p_sold.minute >= '{{project_start_date}}'
+    and p_sold.minute >= TIMESTAMP '{{project_start_date}}'
     {% endif %}
     {% if is_incremental() %}
-    and p_sold.minute >= date_trunc("day", now() - interval '1 week')
+    and p_sold.minute >= date_trunc('day', now() - interval '7' Day)
     {% endif %}
diff --git a/models/lifi/lifi_trades.sql b/models/lifi/lifi_trades.sql
@@ -1,4 +1,5 @@
 {{ config(
+    tags=['dunesql'],
         alias = alias('trades'),
         post_hook='{{ expose_spells(\'["fantom"]\',
                                 "project",
@@ -20,6 +21,7 @@ FROM (
         project,
         version,
         block_date,
+        block_month,
         block_time,
         token_bought_symbol,
         token_sold_symbol,
@@ -44,5 +46,4 @@ FROM (
     UNION ALL
     {% endif %}
     {% endfor %}
-)
-;
+)
diff --git a/models/lifi/lifi_trades_schema.yml b/models/lifi/lifi_trades_schema.yml
@@ -80,4 +80,7 @@ models:
         description: "address of the trace within the call graph forest"
       - &evt_index
         name: evt_index
-        description: "the position of this event log within this transaction"
+        description: "the position of this event log within this transaction"
+      - &block_month
+        name: block_month
+        description: "UTC event block month of each DEX trade"
diff --git a/seeds/dex/aggregator/_schema.yml b/seeds/dex/aggregator/_schema.yml
@@ -5,6 +5,10 @@ seeds:
     config:
       tags: dunesql
       column_types:
+        blockchain: varchar
+        project: varchar
+        version: varchar
+        evt_index: int
         block_date: date
         tx_hash: varbinary
         token_bought_address: varbinary

diff --git a/seeds/dex/aggregator/dex_aggregator_seed.csv b/seeds/dex/aggregator/dex_aggregator_seed.csv
@@ -53,10 +53,10 @@ polygon,paraswap,5,2023-05-05,0xc5efc6c65f70fbb0c0506ccc15d21363a4072f1c7c3ff195
 polygon,paraswap,5,2023-05-05,0x7ecf1b5fc275f1e3ab62a2ea7d8b113f496f7d015a4bbafc88ea487d339e6b03,485,,0x403e967b044d4be25170310157cb1a4bf10bdd0f,,0x6a3e7c3c6ef65ee26975b12293ca1aad7e1daed2,
 polygon,paraswap,5,2023-05-05,0xdda577c3cbe7b211953e465fe8a849a7fb9c483aa6233b5021b610cc2c9291b7,15,,0xc2132d05d31c914a87c6611c10748aeb04b58e8f,74.908523,0xb9a55f61ddfb096dac8f4c120cd65b80df9115f8,
 polygon,paraswap,5,2023-05-05,0x78af6d6b13e3361702d96a327a6f3ac5d8e03852fefb5c4f672d4e1e92f8b201,13,,0x2791bca1f2de4661ed88a30c99a7a9449aa84174,10.039327,0x7ceb23fd6bc0add59e62ac25578270cff1b9f619,0.005292875
-fantom,lifi,2,2023-01-25,0x4dac456aa06573664ab740d7e0f24f010eac09b20567ffe7b2c752f596009d1d,408,,0x04068da6c83afcfa0e13ba15a6696662335d5b75,75.637289,0x049d68029688eabf473097a2fc38ef61633a3c7a,75.6488
-fantom,lifi,2,2023-01-25,0xb1fd0136c59abd5c03670a8473db21aff86d72950343e3feae8a423e0fcce50b,15,,0x04068da6c83afcfa0e13ba15a6696662335d5b75,175.451026,0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83,439.12
-fantom,lifi,2,2023-01-25,0x38b600b3239705220334188a4c7cba04e8600030ea89e2559b951c131d9f9f9d,24,,0x04068da6c83afcfa0e13ba15a6696662335d5b75,75.661808,0x049d68029688eabf473097a2fc38ef61633a3c7a,75.6744
-fantom,lifi,2,2023-01-25,0x47dbfbce72781d9d17c6f85c75c439d14f5b168a2660c2dbac70cba322ea329a,260,,0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83,34.76518051250479,0x74e23df9110aa9ea0b6ff2faee01e740ca1c642e,14
+fantom,lifi,2,2023-01-25,0x4dac456aa06573664ab740d7e0f24f010eac09b20567ffe7b2c752f596009d1d,408,-1,0x04068da6c83afcfa0e13ba15a6696662335d5b75,75.637289,0x049d68029688eabf473097a2fc38ef61633a3c7a,75.6488
+fantom,lifi,2,2023-01-25,0xb1fd0136c59abd5c03670a8473db21aff86d72950343e3feae8a423e0fcce50b,15,-1,0x04068da6c83afcfa0e13ba15a6696662335d5b75,175.451026,0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83,439.12
+fantom,lifi,2,2023-01-25,0x38b600b3239705220334188a4c7cba04e8600030ea89e2559b951c131d9f9f9d,24,-1,0x04068da6c83afcfa0e13ba15a6696662335d5b75,75.661808,0x049d68029688eabf473097a2fc38ef61633a3c7a,75.6744
+fantom,lifi,2,2023-01-25,0x47dbfbce72781d9d17c6f85c75c439d14f5b168a2660c2dbac70cba322ea329a,260,-1,0x21be370d5312f44cb42ce377bc9b8a0cef1a4c83,34.76518051250479,0x74e23df9110aa9ea0b6ff2faee01e740ca1c642e,14
 avalanche_c,odos,1,2023-01-18,0x2b315350cbc1687f12976e66c0a7b7c9d5dbaca94c3b1ab5fa1a73ca359b523e,11,,0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7,5.000449073174419283,0x2b2c81e08f1af8835a78bb2a90ae924ace0ea4be,4.716087894336552324
 avalanche_c,odos,1,2023-01-18,0x7b4b4d34804f1302ba0c5eeb402af7100adb1673e90de7939263e01419201dd1,17,,0xb97ef9ef8734c71904d8002f8b6bc66dd9c48a6e,1.099939,0xa7d7079b0fead91f3e65f86e8915cb59c1a4c664,1.1
 ethereum,1inch,1,2019-06-07,0x001d8b88e6acecdd3e9102e011905b51c4223687d5a76feb32927cf9469890ee,-1,,0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee,0.0117241246838387,0x595832f8fc6bf59c85c527fec3740a1b7a361269,25.915473
@@ -105,4 +105,4 @@ optimism,DODO,0,2022-06-02,0x4326310124907a6c4c53fc8778d8bdcd30c3f06fa46edd72125
 ethereum,tokenlon,5,2021-06-25,0xede75bd74812fc682e62f03190abe22d2fdb11e1c3cf6cc3e29b1baea515f43b,304,,0xdac17f958d2ee523a2206206994597c13d831ec7,27.967009,0xe4815ae53b124e7263f08dcdbbb757d41ed658c6,48
 ethereum,tokenlon,5,2022-12-20,0xb68614d5fdcefcb9ea869dfe0a1381f115d3f8077bf82ab6517ebbb0f4772588,389,,0xdac17f958d2ee523a2206206994597c13d831ec7,47.083995,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.039
 ethereum,tokenlon,5,2021-05-24,0xbe1eda9d0666fd82196e45134909e55a504c6001a43ebc5eadfd55f9dbf8fc1f,101,,0xdac17f958d2ee523a2206206994597c13d831ec7,12905.054001,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,5
-ethereum,tokenlon,5,2022-10-18,0xc168eb7ad27401aeea77e7b1c7f4dc77f8ee2453de1baec2cd3dea32c9e09c3c,384,,0xdac17f958d2ee523a2206206994597c13d831ec7,9806.7654,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7.47
+ethereum,tokenlon,5,2022-10-18,0xc168eb7ad27401aeea77e7b1c7f4dc77f8ee2453de1baec2cd3dea32c9e09c3c,384,,0xdac17f958d2ee523a2206206994597c13d831ec7,9806.7654,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7.47